2024-11-24 23:35:06,356 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-24 23:35:06,371 main DEBUG Took 0.013250 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-24 23:35:06,372 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-24 23:35:06,373 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-24 23:35:06,374 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-24 23:35:06,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,384 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-24 23:35:06,400 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,402 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,403 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,403 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,404 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,404 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,406 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,406 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,407 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,407 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,408 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,408 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,409 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,409 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,410 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,410 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,411 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,411 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,412 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,412 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,413 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,413 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,414 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,414 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 23:35:06,415 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,415 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-24 23:35:06,417 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 23:35:06,419 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-24 23:35:06,421 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-24 23:35:06,422 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-24 23:35:06,423 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-24 23:35:06,424 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-24 23:35:06,433 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-24 23:35:06,436 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-24 23:35:06,438 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-24 23:35:06,439 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-24 23:35:06,439 main DEBUG createAppenders(={Console})
2024-11-24 23:35:06,440 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-24 23:35:06,441 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-24 23:35:06,441 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-24 23:35:06,441 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-24 23:35:06,442 main DEBUG OutputStream closed
2024-11-24 23:35:06,442 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-24 23:35:06,442 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-24 23:35:06,442 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-24 23:35:06,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-24 23:35:06,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-24 23:35:06,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-24 23:35:06,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-24 23:35:06,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-24 23:35:06,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-24 23:35:06,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-24 23:35:06,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-24 23:35:06,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-24 23:35:06,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-24 23:35:06,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-24 23:35:06,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-24 23:35:06,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-24 23:35:06,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-24 23:35:06,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-24 23:35:06,517 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-24 23:35:06,517 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-24 23:35:06,517 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-24 23:35:06,519 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-24 23:35:06,520 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-24 23:35:06,520 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-24 23:35:06,521 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-24T23:35:06,828 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09
2024-11-24 23:35:06,832 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-24 23:35:06,833 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-24T23:35:06,844 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-24T23:35:06,882 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=397, ProcessCount=11, AvailableMemoryMB=3420
2024-11-24T23:35:06,885 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T23:35:06,899 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f, deleteOnExit=true
2024-11-24T23:35:06,900 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T23:35:06,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/test.cache.data in system properties and HBase conf
2024-11-24T23:35:06,902 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T23:35:06,902 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/hadoop.log.dir in system properties and HBase conf
2024-11-24T23:35:06,903 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T23:35:06,903 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T23:35:06,903 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T23:35:07,005 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-24T23:35:07,103 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T23:35:07,107 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:35:07,108 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:35:07,108 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T23:35:07,109 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:35:07,109 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T23:35:07,110 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T23:35:07,110 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:35:07,111 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:35:07,111 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T23:35:07,111 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/nfs.dump.dir in system properties and HBase conf
2024-11-24T23:35:07,112 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/java.io.tmpdir in system properties and HBase conf
2024-11-24T23:35:07,112 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:35:07,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T23:35:07,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T23:35:07,641 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:35:08,300 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-24T23:35:08,374 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2957ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-24T23:35:08,467 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:35:08,562 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:35:08,590 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:35:08,591 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:35:08,592 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:35:08,606 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:35:08,610 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:35:08,611 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:35:08,794 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/java.io.tmpdir/jetty-localhost-38819-hadoop-hdfs-3_4_1-tests_jar-_-any-17981711188806157318/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:35:08,800 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:38819}
2024-11-24T23:35:08,801 INFO  [Time-limited test {}] server.Server(415): Started @3384ms
2024-11-24T23:35:08,829 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:35:09,616 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:35:09,627 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:35:09,629 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:35:09,629 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:35:09,629 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:35:09,633 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:35:09,634 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:35:09,755 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/java.io.tmpdir/jetty-localhost-36591-hadoop-hdfs-3_4_1-tests_jar-_-any-6251350548416244489/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:35:09,755 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:36591}
2024-11-24T23:35:09,756 INFO  [Time-limited test {}] server.Server(415): Started @4339ms
2024-11-24T23:35:09,829 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:35:10,013 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:35:10,026 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:35:10,033 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:35:10,034 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:35:10,034 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:35:10,041 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:35:10,042 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:35:10,184 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/java.io.tmpdir/jetty-localhost-45661-hadoop-hdfs-3_4_1-tests_jar-_-any-17952738708181627699/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:35:10,185 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:45661}
2024-11-24T23:35:10,186 INFO  [Time-limited test {}] server.Server(415): Started @4769ms
2024-11-24T23:35:10,189 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:35:11,774 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f/data/data2/current/BP-77390680-172.17.0.3-1732491307723/current, will proceed with Du for space computation calculation, 
2024-11-24T23:35:11,774 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f/data/data3/current/BP-77390680-172.17.0.3-1732491307723/current, will proceed with Du for space computation calculation, 
2024-11-24T23:35:11,774 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f/data/data4/current/BP-77390680-172.17.0.3-1732491307723/current, will proceed with Du for space computation calculation, 
2024-11-24T23:35:11,774 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f/data/data1/current/BP-77390680-172.17.0.3-1732491307723/current, will proceed with Du for space computation calculation, 
2024-11-24T23:35:11,807 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:35:11,808 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:35:11,861 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2dc2458d70340557 with lease ID 0x94db936800bb5607: Processing first storage report for DS-90180d2c-9aaf-4de0-ae67-2693fd40f086 from datanode DatanodeRegistration(127.0.0.1:44103, datanodeUuid=f7742adb-ba43-44ed-ad66-7617d5b302f9, infoPort=46757, infoSecurePort=0, ipcPort=34177, storageInfo=lv=-57;cid=testClusterID;nsid=1170831972;c=1732491307723)
2024-11-24T23:35:11,863 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2dc2458d70340557 with lease ID 0x94db936800bb5607: from storage DS-90180d2c-9aaf-4de0-ae67-2693fd40f086 node DatanodeRegistration(127.0.0.1:44103, datanodeUuid=f7742adb-ba43-44ed-ad66-7617d5b302f9, infoPort=46757, infoSecurePort=0, ipcPort=34177, storageInfo=lv=-57;cid=testClusterID;nsid=1170831972;c=1732491307723), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T23:35:11,863 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e172a0fce4c9cc8 with lease ID 0x94db936800bb5606: Processing first storage report for DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24 from datanode DatanodeRegistration(127.0.0.1:42031, datanodeUuid=2be3d9de-b3da-4c26-917b-6177c25a15a9, infoPort=45493, infoSecurePort=0, ipcPort=33055, storageInfo=lv=-57;cid=testClusterID;nsid=1170831972;c=1732491307723)
2024-11-24T23:35:11,863 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e172a0fce4c9cc8 with lease ID 0x94db936800bb5606: from storage DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24 node DatanodeRegistration(127.0.0.1:42031, datanodeUuid=2be3d9de-b3da-4c26-917b-6177c25a15a9, infoPort=45493, infoSecurePort=0, ipcPort=33055, storageInfo=lv=-57;cid=testClusterID;nsid=1170831972;c=1732491307723), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T23:35:11,863 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2dc2458d70340557 with lease ID 0x94db936800bb5607: Processing first storage report for DS-a3004b02-ac8c-42c7-9f0e-c2cd8a7bb146 from datanode DatanodeRegistration(127.0.0.1:44103, datanodeUuid=f7742adb-ba43-44ed-ad66-7617d5b302f9, infoPort=46757, infoSecurePort=0, ipcPort=34177, storageInfo=lv=-57;cid=testClusterID;nsid=1170831972;c=1732491307723)
2024-11-24T23:35:11,864 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2dc2458d70340557 with lease ID 0x94db936800bb5607: from storage DS-a3004b02-ac8c-42c7-9f0e-c2cd8a7bb146 node DatanodeRegistration(127.0.0.1:44103, datanodeUuid=f7742adb-ba43-44ed-ad66-7617d5b302f9, infoPort=46757, infoSecurePort=0, ipcPort=34177, storageInfo=lv=-57;cid=testClusterID;nsid=1170831972;c=1732491307723), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:35:11,864 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e172a0fce4c9cc8 with lease ID 0x94db936800bb5606: Processing first storage report for DS-b677728f-4406-4c4f-a5ca-edb06824b072 from datanode DatanodeRegistration(127.0.0.1:42031, datanodeUuid=2be3d9de-b3da-4c26-917b-6177c25a15a9, infoPort=45493, infoSecurePort=0, ipcPort=33055, storageInfo=lv=-57;cid=testClusterID;nsid=1170831972;c=1732491307723)
2024-11-24T23:35:11,864 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e172a0fce4c9cc8 with lease ID 0x94db936800bb5606: from storage DS-b677728f-4406-4c4f-a5ca-edb06824b072 node DatanodeRegistration(127.0.0.1:42031, datanodeUuid=2be3d9de-b3da-4c26-917b-6177c25a15a9, infoPort=45493, infoSecurePort=0, ipcPort=33055, storageInfo=lv=-57;cid=testClusterID;nsid=1170831972;c=1732491307723), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T23:35:11,903 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09
2024-11-24T23:35:11,977 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f/zookeeper_0, clientPort=63184, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T23:35:11,986 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63184
2024-11-24T23:35:11,998 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:35:12,001 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:35:12,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:35:12,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:35:12,635 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec with version=8
2024-11-24T23:35:12,635 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/hbase-staging
2024-11-24T23:35:12,718 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-24T23:35:12,919 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:35:12,928 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:35:12,929 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:35:12,933 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:35:12,934 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:35:12,934 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:35:13,083 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T23:35:13,148 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-24T23:35:13,157 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-24T23:35:13,161 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:35:13,182 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 97307 (auto-detected)
2024-11-24T23:35:13,183 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected)
2024-11-24T23:35:13,201 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40669
2024-11-24T23:35:13,224 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40669 connecting to ZooKeeper ensemble=127.0.0.1:63184
2024-11-24T23:35:13,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:406690x0, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:35:13,331 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40669-0x1016f3377010000 connected
2024-11-24T23:35:13,432 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:35:13,435 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:35:13,444 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:35:13,449 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec, hbase.cluster.distributed=false
2024-11-24T23:35:13,478 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:35:13,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40669
2024-11-24T23:35:13,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40669
2024-11-24T23:35:13,488 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40669
2024-11-24T23:35:13,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40669
2024-11-24T23:35:13,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40669
2024-11-24T23:35:13,600 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:35:13,602 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:35:13,603 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:35:13,603 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:35:13,603 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:35:13,604 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:35:13,607 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T23:35:13,611 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:35:13,612 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45543
2024-11-24T23:35:13,614 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45543 connecting to ZooKeeper ensemble=127.0.0.1:63184
2024-11-24T23:35:13,615 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:35:13,622 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:35:13,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:455430x0, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:35:13,671 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:455430x0, quorum=127.0.0.1:63184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:35:13,671 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45543-0x1016f3377010001 connected
2024-11-24T23:35:13,676 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T23:35:13,686 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T23:35:13,689 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T23:35:13,694 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:35:13,695 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45543
2024-11-24T23:35:13,695 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45543
2024-11-24T23:35:13,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45543
2024-11-24T23:35:13,703 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45543
2024-11-24T23:35:13,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45543
2024-11-24T23:35:13,724 DEBUG [M:0;ba7f32532130:40669 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ba7f32532130:40669
2024-11-24T23:35:13,725 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ba7f32532130,40669,1732491312763
2024-11-24T23:35:13,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:35:13,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:35:13,777 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ba7f32532130,40669,1732491312763
2024-11-24T23:35:13,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:13,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T23:35:13,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:13,840 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T23:35:13,841 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ba7f32532130,40669,1732491312763 from backup master directory
2024-11-24T23:35:13,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:35:13,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ba7f32532130,40669,1732491312763
2024-11-24T23:35:13,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:35:13,893 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:35:13,893 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ba7f32532130,40669,1732491312763
2024-11-24T23:35:13,895 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-24T23:35:13,897 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-24T23:35:13,966 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/hbase.id] with ID: a97a0db2-cdec-4baa-bd79-a01db4e97cd1
2024-11-24T23:35:13,967 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/.tmp/hbase.id
2024-11-24T23:35:13,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:35:13,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:35:13,984 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/.tmp/hbase.id]:[hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/hbase.id]
2024-11-24T23:35:14,038 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:35:14,044 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T23:35:14,063 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-11-24T23:35:14,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:14,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:14,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:35:14,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:35:14,161 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:35:14,163 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T23:35:14,168 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:35:14,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:35:14,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:35:14,218 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store
2024-11-24T23:35:14,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:35:14,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:35:14,247 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-24T23:35:14,251 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:35:14,253 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:35:14,253 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:35:14,254 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:35:14,255 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:35:14,256 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:35:14,256 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:35:14,258 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491314253Disabling compacts and flushes for region at 1732491314253Disabling writes for close at 1732491314256 (+3 ms)Writing region close event to WAL at 1732491314256Closed at 1732491314256
2024-11-24T23:35:14,260 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/.initializing
2024-11-24T23:35:14,261 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/WALs/ba7f32532130,40669,1732491312763
2024-11-24T23:35:14,287 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C40669%2C1732491312763, suffix=, logDir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/WALs/ba7f32532130,40669,1732491312763, archiveDir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/oldWALs, maxLogs=10
2024-11-24T23:35:14,297 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C40669%2C1732491312763.1732491314293
2024-11-24T23:35:14,316 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/WALs/ba7f32532130,40669,1732491312763/ba7f32532130%2C40669%2C1732491312763.1732491314293
2024-11-24T23:35:14,324 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46757:46757),(127.0.0.1/127.0.0.1:45493:45493)]
2024-11-24T23:35:14,325 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:35:14,326 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:35:14,328 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:35:14,329 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:35:14,364 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:35:14,386 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T23:35:14,389 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:14,391 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:35:14,392 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:35:14,395 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T23:35:14,396 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:14,397 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:35:14,397 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:35:14,400 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T23:35:14,400 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:14,401 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:35:14,401 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:35:14,404 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T23:35:14,404 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:14,405 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:35:14,406 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:35:14,409 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:35:14,410 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:35:14,416 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:35:14,416 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:35:14,420 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T23:35:14,423 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:35:14,426 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:35:14,427 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=833815, jitterRate=0.06025080382823944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T23:35:14,434 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732491314340Initializing all the Stores at 1732491314342 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491314343 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491314344 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491314344Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491314344Cleaning up temporary data from old regions at 1732491314416 (+72 ms)Region opened successfully at 1732491314434 (+18 ms)
2024-11-24T23:35:14,435 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T23:35:14,469 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ae66e14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:35:14,501 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T23:35:14,514 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T23:35:14,515 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T23:35:14,517 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T23:35:14,519 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-24T23:35:14,524 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-24T23:35:14,524 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T23:35:14,548 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T23:35:14,556 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T23:35:14,588 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T23:35:14,590 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T23:35:14,592 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T23:35:14,658 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T23:35:14,661 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T23:35:14,664 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T23:35:14,739 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T23:35:14,743 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T23:35:14,792 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T23:35:14,815 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T23:35:14,870 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T23:35:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:35:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:35:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:14,950 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ba7f32532130,40669,1732491312763, sessionid=0x1016f3377010000, setting cluster-up flag (Was=false)
2024-11-24T23:35:15,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:15,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:15,367 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T23:35:15,370 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,40669,1732491312763
2024-11-24T23:35:15,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:15,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:15,435 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T23:35:15,440 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,40669,1732491312763
2024-11-24T23:35:15,446 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T23:35:15,517 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(746): ClusterId : a97a0db2-cdec-4baa-bd79-a01db4e97cd1
2024-11-24T23:35:15,521 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T23:35:15,533 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T23:35:15,537 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T23:35:15,537 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T23:35:15,547 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T23:35:15,552 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T23:35:15,553 DEBUG [RS:0;ba7f32532130:45543 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69a8bbe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:35:15,555 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T23:35:15,563 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ba7f32532130,40669,1732491312763
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T23:35:15,573 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:35:15,574 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:35:15,574 DEBUG [RS:0;ba7f32532130:45543 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ba7f32532130:45543
2024-11-24T23:35:15,574 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:35:15,574 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:35:15,574 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ba7f32532130:0, corePoolSize=10, maxPoolSize=10
2024-11-24T23:35:15,575 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,575 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:35:15,575 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,577 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T23:35:15,577 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T23:35:15,577 DEBUG [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T23:35:15,580 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba7f32532130,40669,1732491312763 with port=45543, startcode=1732491313562
2024-11-24T23:35:15,581 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732491345580
2024-11-24T23:35:15,582 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:35:15,582 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T23:35:15,583 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T23:35:15,584 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T23:35:15,588 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T23:35:15,589 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:15,589 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T23:35:15,589 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T23:35:15,590 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T23:35:15,589 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T23:35:15,590 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,597 DEBUG [RS:0;ba7f32532130:45543 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T23:35:15,601 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T23:35:15,602 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T23:35:15,603 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T23:35:15,607 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T23:35:15,607 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T23:35:15,616 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491315609,5,FailOnTimeoutGroup]
2024-11-24T23:35:15,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:35:15,618 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:35:15,622 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491315616,5,FailOnTimeoutGroup]
2024-11-24T23:35:15,622 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,622 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T23:35:15,624 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,624 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,628 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T23:35:15,629 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec
2024-11-24T23:35:15,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:35:15,661 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:35:15,664 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:35:15,679 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:35:15,683 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:35:15,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:15,684 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:35:15,685 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:35:15,688 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:35:15,689 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:15,690 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:35:15,691 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:35:15,695 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:35:15,695 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:15,696 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43487, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T23:35:15,696 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:35:15,697 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:35:15,701 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:35:15,702 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:15,704 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:35:15,704 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40669 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba7f32532130,45543,1732491313562
2024-11-24T23:35:15,704 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:35:15,706 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740
2024-11-24T23:35:15,707 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40669 {}] master.ServerManager(517): Registering regionserver=ba7f32532130,45543,1732491313562
2024-11-24T23:35:15,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740
2024-11-24T23:35:15,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:35:15,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:35:15,714 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:35:15,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:35:15,724 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:35:15,725 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=707031, jitterRate=-0.1009637862443924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:35:15,727 DEBUG [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec
2024-11-24T23:35:15,727 DEBUG [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43777
2024-11-24T23:35:15,727 DEBUG [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T23:35:15,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732491315668Initializing all the Stores at 1732491315671 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491315671Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491315672 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491315672Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491315672Cleaning up temporary data from old regions at 1732491315712 (+40 ms)Region opened successfully at 1732491315730 (+18 ms)
2024-11-24T23:35:15,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:35:15,730 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:35:15,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:35:15,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:35:15,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:35:15,733 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:35:15,733 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491315730Disabling compacts and flushes for region at 1732491315730Disabling writes for close at 1732491315730Writing region close event to WAL at 1732491315732 (+2 ms)Closed at 1732491315733 (+1 ms)
2024-11-24T23:35:15,736 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:35:15,737 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T23:35:15,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:35:15,741 DEBUG [RS:0;ba7f32532130:45543 {}] zookeeper.ZKUtil(111): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba7f32532130,45543,1732491313562
2024-11-24T23:35:15,741 WARN  [RS:0;ba7f32532130:45543 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:35:15,741 INFO  [RS:0;ba7f32532130:45543 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:35:15,742 DEBUG [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562
2024-11-24T23:35:15,743 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T23:35:15,743 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba7f32532130,45543,1732491313562]
2024-11-24T23:35:15,752 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:35:15,756 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T23:35:15,773 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T23:35:15,788 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T23:35:15,792 INFO  [RS:0;ba7f32532130:45543 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T23:35:15,792 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,793 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T23:35:15,798 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T23:35:15,799 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,800 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,800 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,800 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,800 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,800 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,800 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:35:15,801 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,801 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,801 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,801 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,801 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,801 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:35:15,801 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:35:15,801 DEBUG [RS:0;ba7f32532130:45543 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:35:15,802 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,802 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,803 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,803 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,803 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,803 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,45543,1732491313562-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:35:15,821 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T23:35:15,823 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,45543,1732491313562-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,823 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,824 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.Replication(171): ba7f32532130,45543,1732491313562 started
2024-11-24T23:35:15,842 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:15,843 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(1482): Serving as ba7f32532130,45543,1732491313562, RpcServer on ba7f32532130/172.17.0.3:45543, sessionid=0x1016f3377010001
2024-11-24T23:35:15,843 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T23:35:15,844 DEBUG [RS:0;ba7f32532130:45543 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba7f32532130,45543,1732491313562
2024-11-24T23:35:15,844 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,45543,1732491313562'
2024-11-24T23:35:15,844 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T23:35:15,845 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T23:35:15,846 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T23:35:15,846 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T23:35:15,846 DEBUG [RS:0;ba7f32532130:45543 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba7f32532130,45543,1732491313562
2024-11-24T23:35:15,846 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,45543,1732491313562'
2024-11-24T23:35:15,846 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T23:35:15,847 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T23:35:15,848 DEBUG [RS:0;ba7f32532130:45543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T23:35:15,848 INFO  [RS:0;ba7f32532130:45543 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T23:35:15,848 INFO  [RS:0;ba7f32532130:45543 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T23:35:15,907 WARN  [ba7f32532130:40669 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T23:35:15,956 INFO  [RS:0;ba7f32532130:45543 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C45543%2C1732491313562, suffix=, logDir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562, archiveDir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/oldWALs, maxLogs=32
2024-11-24T23:35:15,959 INFO  [RS:0;ba7f32532130:45543 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45543%2C1732491313562.1732491315958
2024-11-24T23:35:15,967 INFO  [RS:0;ba7f32532130:45543 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491315958
2024-11-24T23:35:15,969 DEBUG [RS:0;ba7f32532130:45543 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45493:45493),(127.0.0.1/127.0.0.1:46757:46757)]
2024-11-24T23:35:16,162 DEBUG [ba7f32532130:40669 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T23:35:16,179 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba7f32532130,45543,1732491313562
2024-11-24T23:35:16,185 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,45543,1732491313562, state=OPENING
2024-11-24T23:35:16,240 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T23:35:16,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:16,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:35:16,251 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:35:16,251 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:35:16,253 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:35:16,256 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,45543,1732491313562}]
2024-11-24T23:35:16,430 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T23:35:16,434 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49035, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T23:35:16,448 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T23:35:16,449 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:35:16,453 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C45543%2C1732491313562.meta, suffix=.meta, logDir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562, archiveDir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/oldWALs, maxLogs=32
2024-11-24T23:35:16,457 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45543%2C1732491313562.meta.1732491316456.meta
2024-11-24T23:35:16,470 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.meta.1732491316456.meta
2024-11-24T23:35:16,476 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46757:46757),(127.0.0.1/127.0.0.1:45493:45493)]
2024-11-24T23:35:16,478 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:35:16,481 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T23:35:16,484 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T23:35:16,490 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T23:35:16,495 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T23:35:16,496 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:35:16,496 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T23:35:16,496 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T23:35:16,500 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:35:16,503 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:35:16,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:16,505 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:35:16,505 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:35:16,507 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:35:16,508 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:16,509 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:35:16,509 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:35:16,512 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:35:16,512 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:16,514 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:35:16,514 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:35:16,516 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:35:16,517 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:16,518 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:35:16,518 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:35:16,520 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740
2024-11-24T23:35:16,524 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740
2024-11-24T23:35:16,527 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:35:16,527 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:35:16,529 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:35:16,532 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:35:16,535 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=718290, jitterRate=-0.08664783835411072}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:35:16,535 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T23:35:16,537 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732491316497Writing region info on filesystem at 1732491316497Initializing all the Stores at 1732491316499 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491316500 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491316500Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491316500Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491316500Cleaning up temporary data from old regions at 1732491316527 (+27 ms)Running coprocessor post-open hooks at 1732491316535 (+8 ms)Region opened successfully at 1732491316537 (+2 ms)
2024-11-24T23:35:16,546 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732491316422
2024-11-24T23:35:16,561 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T23:35:16,562 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T23:35:16,564 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,45543,1732491313562
2024-11-24T23:35:16,566 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,45543,1732491313562, state=OPEN
2024-11-24T23:35:16,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:35:16,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:35:16,643 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:35:16,643 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:35:16,644 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba7f32532130,45543,1732491313562
2024-11-24T23:35:16,649 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T23:35:16,650 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,45543,1732491313562 in 389 msec
2024-11-24T23:35:16,658 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T23:35:16,659 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 908 msec
2024-11-24T23:35:16,660 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:35:16,660 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T23:35:16,677 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:35:16,678 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,45543,1732491313562, seqNum=-1]
2024-11-24T23:35:16,699 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:35:16,701 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41219, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:35:16,722 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2450 sec
2024-11-24T23:35:16,722 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732491316722, completionTime=-1
2024-11-24T23:35:16,725 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T23:35:16,725 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T23:35:16,750 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T23:35:16,750 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732491376750
2024-11-24T23:35:16,750 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732491436750
2024-11-24T23:35:16,750 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec
2024-11-24T23:35:16,753 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,40669,1732491312763-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:16,753 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,40669,1732491312763-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:16,753 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,40669,1732491312763-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:16,755 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ba7f32532130:40669, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:16,755 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:16,756 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:16,761 DEBUG [master/ba7f32532130:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T23:35:16,782 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.888sec
2024-11-24T23:35:16,783 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T23:35:16,785 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T23:35:16,786 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T23:35:16,786 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T23:35:16,786 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T23:35:16,787 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,40669,1732491312763-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:35:16,788 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,40669,1732491312763-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T23:35:16,798 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T23:35:16,798 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T23:35:16,799 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,40669,1732491312763-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:35:16,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cb686a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:35:16,852 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-24T23:35:16,852 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-24T23:35:16,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ba7f32532130,40669,-1 for getting cluster id
2024-11-24T23:35:16,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T23:35:16,878 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a97a0db2-cdec-4baa-bd79-a01db4e97cd1'
2024-11-24T23:35:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T23:35:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a97a0db2-cdec-4baa-bd79-a01db4e97cd1"

2024-11-24T23:35:16,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61568b2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:35:16,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ba7f32532130,40669,-1]
2024-11-24T23:35:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T23:35:16,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:35:16,894 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T23:35:16,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fb6022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:35:16,897 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:35:16,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,45543,1732491313562, seqNum=-1]
2024-11-24T23:35:16,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:35:16,907 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:35:16,926 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ba7f32532130,40669,1732491312763
2024-11-24T23:35:16,926 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:35:16,933 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T23:35:16,936 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-24T23:35:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ba7f32532130,40669,1732491312763
2024-11-24T23:35:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ae36ead
2024-11-24T23:35:16,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-24T23:35:16,949 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-24T23:35:16,952 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-24T23:35:16,952 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-24T23:35:16,955 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:35:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-24T23:35:16,966 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-24T23:35:16,969 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-24T23:35:16,969 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:16,972 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-24T23:35:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T23:35:17,004 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741835_1011 (size=389)
2024-11-24T23:35:17,004 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741835_1011 (size=389)
2024-11-24T23:35:17,009 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 286f94c7df1b11f1e41ec361e8d9e2ab, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec
2024-11-24T23:35:17,023 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741836_1012 (size=72)
2024-11-24T23:35:17,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741836_1012 (size=72)
2024-11-24T23:35:17,025 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:35:17,025 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 286f94c7df1b11f1e41ec361e8d9e2ab, disabling compactions & flushes
2024-11-24T23:35:17,026 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:35:17,026 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:35:17,026 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab. after waiting 0 ms
2024-11-24T23:35:17,026 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:35:17,026 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:35:17,026 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 286f94c7df1b11f1e41ec361e8d9e2ab:
Waiting for close lock at 1732491317025Disabling compacts and flushes for region at 1732491317025Disabling writes for close at 1732491317026 (+1 ms)Writing region close event to WAL at 1732491317026Closed at 1732491317026
2024-11-24T23:35:17,028 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-24T23:35:17,032 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1732491317028"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732491317028"}]},"ts":"1732491317028"}
2024-11-24T23:35:17,038 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-24T23:35:17,040 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-24T23:35:17,044 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732491317041"}]},"ts":"1732491317041"}
2024-11-24T23:35:17,053 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-24T23:35:17,055 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=286f94c7df1b11f1e41ec361e8d9e2ab, ASSIGN}]
2024-11-24T23:35:17,058 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=286f94c7df1b11f1e41ec361e8d9e2ab, ASSIGN
2024-11-24T23:35:17,060 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=286f94c7df1b11f1e41ec361e8d9e2ab, ASSIGN; state=OFFLINE, location=ba7f32532130,45543,1732491313562; forceNewPlan=false, retain=false
2024-11-24T23:35:17,212 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=286f94c7df1b11f1e41ec361e8d9e2ab, regionState=OPENING, regionLocation=ba7f32532130,45543,1732491313562
2024-11-24T23:35:17,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=286f94c7df1b11f1e41ec361e8d9e2ab, ASSIGN because future has completed
2024-11-24T23:35:17,220 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 286f94c7df1b11f1e41ec361e8d9e2ab, server=ba7f32532130,45543,1732491313562}]
2024-11-24T23:35:17,382 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:35:17,382 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 286f94c7df1b11f1e41ec361e8d9e2ab, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:35:17,383 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:17,383 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:35:17,383 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:17,383 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:17,385 INFO  [StoreOpener-286f94c7df1b11f1e41ec361e8d9e2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 286f94c7df1b11f1e41ec361e8d9e2ab 
2024-11-24T23:35:17,388 INFO  [StoreOpener-286f94c7df1b11f1e41ec361e8d9e2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 286f94c7df1b11f1e41ec361e8d9e2ab columnFamilyName info
2024-11-24T23:35:17,388 DEBUG [StoreOpener-286f94c7df1b11f1e41ec361e8d9e2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:35:17,389 INFO  [StoreOpener-286f94c7df1b11f1e41ec361e8d9e2ab-1 {}] regionserver.HStore(327): Store=286f94c7df1b11f1e41ec361e8d9e2ab/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:35:17,390 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:17,392 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:17,392 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:17,393 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:17,393 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:17,396 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:17,400 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:35:17,402 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 286f94c7df1b11f1e41ec361e8d9e2ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=798293, jitterRate=0.015082255005836487}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T23:35:17,402 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:17,403 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 286f94c7df1b11f1e41ec361e8d9e2ab:
Running coprocessor pre-open hook at 1732491317383Writing region info on filesystem at 1732491317383Initializing all the Stores at 1732491317385 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491317385Cleaning up temporary data from old regions at 1732491317393 (+8 ms)Running coprocessor post-open hooks at 1732491317402 (+9 ms)Region opened successfully at 1732491317403 (+1 ms)
2024-11-24T23:35:17,406 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab., pid=6, masterSystemTime=1732491317374
2024-11-24T23:35:17,411 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:35:17,411 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:35:17,413 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=286f94c7df1b11f1e41ec361e8d9e2ab, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,45543,1732491313562
2024-11-24T23:35:17,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 286f94c7df1b11f1e41ec361e8d9e2ab, server=ba7f32532130,45543,1732491313562 because future has completed
2024-11-24T23:35:17,428 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-24T23:35:17,430 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 286f94c7df1b11f1e41ec361e8d9e2ab, server=ba7f32532130,45543,1732491313562 in 203 msec
2024-11-24T23:35:17,433 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-24T23:35:17,434 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=286f94c7df1b11f1e41ec361e8d9e2ab, ASSIGN in 373 msec
2024-11-24T23:35:17,435 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-24T23:35:17,436 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732491317435"}]},"ts":"1732491317435"}
2024-11-24T23:35:17,440 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-24T23:35:17,442 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-24T23:35:17,446 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 484 msec
2024-11-24T23:35:22,005 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-24T23:35:22,062 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-24T23:35:22,063 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-24T23:35:23,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T23:35:23,145 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-24T23:35:23,147 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-24T23:35:23,147 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-24T23:35:23,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:35:23,149 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-24T23:35:23,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-24T23:35:23,150 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-24T23:35:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T23:35:27,032 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-24T23:35:27,036 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-24T23:35:27,043 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-24T23:35:27,044 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:35:27,045 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45543%2C1732491313562.1732491327045
2024-11-24T23:35:27,070 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:27,070 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:27,070 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:27,070 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:27,070 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:27,071 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491315958 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491327045
2024-11-24T23:35:27,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741833_1009 (size=451)
2024-11-24T23:35:27,075 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741833_1009 (size=451)
2024-11-24T23:35:27,077 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491315958 to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/oldWALs/ba7f32532130%2C45543%2C1732491313562.1732491315958
2024-11-24T23:35:27,081 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45493:45493),(127.0.0.1/127.0.0.1:46757:46757)]
2024-11-24T23:35:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab., hostname=ba7f32532130,45543,1732491313562, seqNum=2]
2024-11-24T23:35:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45543 {}] regionserver.HRegion(8855): Flush requested on 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:39,132 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 286f94c7df1b11f1e41ec361e8d9e2ab 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T23:35:39,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/a5b0f38fffd8487f814c607e23a184ca is 1080, key is row0001/info:/1732491327093/Put/seqid=0
2024-11-24T23:35:39,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741838_1014 (size=12509)
2024-11-24T23:35:39,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741838_1014 (size=12509)
2024-11-24T23:35:39,200 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/a5b0f38fffd8487f814c607e23a184ca
2024-11-24T23:35:39,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/a5b0f38fffd8487f814c607e23a184ca as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/a5b0f38fffd8487f814c607e23a184ca
2024-11-24T23:35:39,256 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/a5b0f38fffd8487f814c607e23a184ca, entries=7, sequenceid=11, filesize=12.2 K
2024-11-24T23:35:39,264 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 286f94c7df1b11f1e41ec361e8d9e2ab in 131ms, sequenceid=11, compaction requested=false
2024-11-24T23:35:39,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 286f94c7df1b11f1e41ec361e8d9e2ab:

2024-11-24T23:35:41,901 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T23:35:47,149 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45543%2C1732491313562.1732491347149
2024-11-24T23:35:47,367 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 212 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK], DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK]]
2024-11-24T23:35:47,367 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:47,367 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:47,367 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:47,367 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:47,368 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:47,368 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491327045 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491347149
2024-11-24T23:35:47,369 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46757:46757),(127.0.0.1/127.0.0.1:45493:45493)]
2024-11-24T23:35:47,369 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491327045 is not closed yet, will try archiving it next time
2024-11-24T23:35:47,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741837_1013 (size=12399)
2024-11-24T23:35:47,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741837_1013 (size=12399)
2024-11-24T23:35:47,575 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:35:49,782 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:35:51,987 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:35:54,191 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:35:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45543 {}] regionserver.HRegion(8855): Flush requested on 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:35:54,191 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 286f94c7df1b11f1e41ec361e8d9e2ab 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T23:35:54,394 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:35:54,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/5bf1045fec8643e3a9671fe69c7082ff is 1080, key is row0008/info:/1732491341132/Put/seqid=0
2024-11-24T23:35:54,406 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741840_1016 (size=12509)
2024-11-24T23:35:54,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741840_1016 (size=12509)
2024-11-24T23:35:54,407 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/5bf1045fec8643e3a9671fe69c7082ff
2024-11-24T23:35:54,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/5bf1045fec8643e3a9671fe69c7082ff as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/5bf1045fec8643e3a9671fe69c7082ff
2024-11-24T23:35:54,428 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/5bf1045fec8643e3a9671fe69c7082ff, entries=7, sequenceid=21, filesize=12.2 K
2024-11-24T23:35:54,630 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:35:54,630 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 286f94c7df1b11f1e41ec361e8d9e2ab in 439ms, sequenceid=21, compaction requested=false
2024-11-24T23:35:54,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 286f94c7df1b11f1e41ec361e8d9e2ab:

2024-11-24T23:35:54,630 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-24T23:35:54,631 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:35:54,631 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/a5b0f38fffd8487f814c607e23a184ca because midkey is the same as first or last row
2024-11-24T23:35:56,397 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:35:56,801 INFO  [master/ba7f32532130:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-24T23:35:56,801 INFO  [master/ba7f32532130:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-24T23:35:58,601 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:35:58,603 WARN  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:35:58,604 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C45543%2C1732491313562:(num 1732491347149) roll requested
2024-11-24T23:35:58,605 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45543%2C1732491313562.1732491358605
2024-11-24T23:35:58,815 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 207 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:35:58,815 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:58,815 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:58,815 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:58,815 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:58,816 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:35:58,816 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491347149 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491358605
2024-11-24T23:35:58,817 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46757:46757),(127.0.0.1/127.0.0.1:45493:45493)]
2024-11-24T23:35:58,817 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491347149 is not closed yet, will try archiving it next time
2024-11-24T23:35:58,817 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491327045 to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/oldWALs/ba7f32532130%2C45543%2C1732491313562.1732491327045
2024-11-24T23:35:58,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741839_1015 (size=7739)
2024-11-24T23:35:58,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741839_1015 (size=7739)
2024-11-24T23:36:00,805 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:36:02,383 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 286f94c7df1b11f1e41ec361e8d9e2ab, had cached 0 bytes from a total of 25018
2024-11-24T23:36:03,009 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:36:05,213 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:36:07,417 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:36:09,421 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-24T23:36:09,423 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45543%2C1732491313562.1732491369422
2024-11-24T23:36:11,901 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T23:36:14,436 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5007 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:36:14,438 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5007 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK], DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK]]
2024-11-24T23:36:14,439 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C45543%2C1732491313562:(num 1732491369422) roll requested
2024-11-24T23:36:14,439 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:14,439 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:14,439 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:14,439 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:14,440 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:14,440 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491358605 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491369422
2024-11-24T23:36:14,441 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45493:45493),(127.0.0.1/127.0.0.1:46757:46757)]
2024-11-24T23:36:14,441 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491358605 is not closed yet, will try archiving it next time
2024-11-24T23:36:14,442 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45543%2C1732491313562.1732491374442
2024-11-24T23:36:14,443 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741841_1017 (size=4753)
2024-11-24T23:36:14,444 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741841_1017 (size=4753)
2024-11-24T23:36:19,446 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK], DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK]]
2024-11-24T23:36:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45543 {}] regionserver.HRegion(8855): Flush requested on 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:36:19,446 WARN  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK], DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK]]
2024-11-24T23:36:19,447 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 286f94c7df1b11f1e41ec361e8d9e2ab 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T23:36:19,452 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK], DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK]]
2024-11-24T23:36:19,452 WARN  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK], DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK]]
2024-11-24T23:36:21,448 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-24T23:36:24,451 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK], DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK]]
2024-11-24T23:36:24,451 WARN  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK], DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK]]
2024-11-24T23:36:24,452 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:24,452 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:24,453 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:24,453 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:24,454 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:24,455 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491369422 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491374442
2024-11-24T23:36:24,456 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45493:45493),(127.0.0.1/127.0.0.1:46757:46757)]
2024-11-24T23:36:24,457 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491369422 is not closed yet, will try archiving it next time
2024-11-24T23:36:24,457 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C45543%2C1732491313562:(num 1732491374442) roll requested
2024-11-24T23:36:24,458 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45543%2C1732491313562.1732491384457
2024-11-24T23:36:24,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741842_1018 (size=1569)
2024-11-24T23:36:24,459 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741842_1018 (size=1569)
2024-11-24T23:36:24,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/208df59c54bc4b318456b191fbab8fe1 is 1080, key is row0015/info:/1732491356195/Put/seqid=0
2024-11-24T23:36:24,468 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741844_1020 (size=12509)
2024-11-24T23:36:24,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741844_1020 (size=12509)
2024-11-24T23:36:24,469 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/208df59c54bc4b318456b191fbab8fe1
2024-11-24T23:36:24,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/208df59c54bc4b318456b191fbab8fe1 as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/208df59c54bc4b318456b191fbab8fe1
2024-11-24T23:36:24,491 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/208df59c54bc4b318456b191fbab8fe1, entries=7, sequenceid=31, filesize=12.2 K
2024-11-24T23:36:29,473 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5012 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK], DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK]]
2024-11-24T23:36:29,473 WARN  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5012 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK], DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK]]
2024-11-24T23:36:29,492 INFO  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK], DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK]]
2024-11-24T23:36:29,492 WARN  [FSHLog-0-hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec-prefix:ba7f32532130,45543,1732491313562 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42031,DS-d2b78df7-a589-4255-bb8c-3e9963e1fd24,DISK], DatanodeInfoWithStorage[127.0.0.1:44103,DS-90180d2c-9aaf-4de0-ae67-2693fd40f086,DISK]]
2024-11-24T23:36:29,492 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 286f94c7df1b11f1e41ec361e8d9e2ab in 10046ms, sequenceid=31, compaction requested=true
2024-11-24T23:36:29,492 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 286f94c7df1b11f1e41ec361e8d9e2ab:

2024-11-24T23:36:29,493 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,493 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,493 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-24T23:36:29,493 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,493 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:36:29,493 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,493 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/a5b0f38fffd8487f814c607e23a184ca because midkey is the same as first or last row
2024-11-24T23:36:29,493 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491374442 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491384457
2024-11-24T23:36:29,494 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46757:46757),(127.0.0.1/127.0.0.1:45493:45493)]
2024-11-24T23:36:29,494 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491374442 is not closed yet, will try archiving it next time
2024-11-24T23:36:29,494 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C45543%2C1732491313562:(num 1732491384457) roll requested
2024-11-24T23:36:29,495 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491347149 to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/oldWALs/ba7f32532130%2C45543%2C1732491313562.1732491347149
2024-11-24T23:36:29,495 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45543%2C1732491313562.1732491389495
2024-11-24T23:36:29,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 286f94c7df1b11f1e41ec361e8d9e2ab:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:36:29,496 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741843_1019 (size=438)
2024-11-24T23:36:29,497 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741843_1019 (size=438)
2024-11-24T23:36:29,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:36:29,498 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491358605 to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/oldWALs/ba7f32532130%2C45543%2C1732491313562.1732491358605
2024-11-24T23:36:29,498 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:36:29,500 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491369422 to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/oldWALs/ba7f32532130%2C45543%2C1732491313562.1732491369422
2024-11-24T23:36:29,501 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491374442 to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/oldWALs/ba7f32532130%2C45543%2C1732491313562.1732491374442
2024-11-24T23:36:29,502 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:36:29,503 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.HStore(1541): 286f94c7df1b11f1e41ec361e8d9e2ab/info is initiating minor compaction (all files)
2024-11-24T23:36:29,504 INFO  [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 286f94c7df1b11f1e41ec361e8d9e2ab/info in TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:36:29,504 INFO  [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/a5b0f38fffd8487f814c607e23a184ca, hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/5bf1045fec8643e3a9671fe69c7082ff, hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/208df59c54bc4b318456b191fbab8fe1] into tmpdir=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp, totalSize=36.6 K
2024-11-24T23:36:29,505 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5b0f38fffd8487f814c607e23a184ca, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732491327093
2024-11-24T23:36:29,506 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bf1045fec8643e3a9671fe69c7082ff, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1732491341132
2024-11-24T23:36:29,507 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 208df59c54bc4b318456b191fbab8fe1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732491356195
2024-11-24T23:36:29,514 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,514 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,514 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,515 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,515 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,515 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491384457 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491389495
2024-11-24T23:36:29,517 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741845_1021 (size=93)
2024-11-24T23:36:29,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741845_1021 (size=93)
2024-11-24T23:36:29,518 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491384457 to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/oldWALs/ba7f32532130%2C45543%2C1732491313562.1732491384457
2024-11-24T23:36:29,523 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45493:45493),(127.0.0.1/127.0.0.1:46757:46757)]
2024-11-24T23:36:29,524 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45543%2C1732491313562.1732491389524
2024-11-24T23:36:29,540 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,540 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,540 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,540 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,540 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:29,541 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491389495 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/WALs/ba7f32532130,45543,1732491313562/ba7f32532130%2C45543%2C1732491313562.1732491389524
2024-11-24T23:36:29,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741846_1022 (size=1258)
2024-11-24T23:36:29,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741846_1022 (size=1258)
2024-11-24T23:36:29,546 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45493:45493),(127.0.0.1/127.0.0.1:46757:46757)]
2024-11-24T23:36:29,546 INFO  [RS:0;ba7f32532130:45543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 286f94c7df1b11f1e41ec361e8d9e2ab#info#compaction#3 average throughput is 7.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:36:29,547 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/ed78c6dadcc240938641a6feb008238f is 1080, key is row0001/info:/1732491327093/Put/seqid=0
2024-11-24T23:36:29,555 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741848_1024 (size=27710)
2024-11-24T23:36:29,556 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741848_1024 (size=27710)
2024-11-24T23:36:29,567 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/ed78c6dadcc240938641a6feb008238f as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/ed78c6dadcc240938641a6feb008238f
2024-11-24T23:36:29,583 INFO  [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 286f94c7df1b11f1e41ec361e8d9e2ab/info of 286f94c7df1b11f1e41ec361e8d9e2ab into ed78c6dadcc240938641a6feb008238f(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:36:29,584 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 286f94c7df1b11f1e41ec361e8d9e2ab:

2024-11-24T23:36:29,585 INFO  [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab., storeName=286f94c7df1b11f1e41ec361e8d9e2ab/info, priority=13, startTime=1732491389495; duration=0sec
2024-11-24T23:36:29,586 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-24T23:36:29,586 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:36:29,586 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/ed78c6dadcc240938641a6feb008238f because midkey is the same as first or last row
2024-11-24T23:36:29,586 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-24T23:36:29,586 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:36:29,586 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/ed78c6dadcc240938641a6feb008238f because midkey is the same as first or last row
2024-11-24T23:36:29,586 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-24T23:36:29,586 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:36:29,586 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/ed78c6dadcc240938641a6feb008238f because midkey is the same as first or last row
2024-11-24T23:36:29,587 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:36:29,587 DEBUG [RS:0;ba7f32532130:45543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 286f94c7df1b11f1e41ec361e8d9e2ab:info
2024-11-24T23:36:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45543 {}] regionserver.HRegion(8855): Flush requested on 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:36:41,568 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 286f94c7df1b11f1e41ec361e8d9e2ab 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T23:36:41,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/321e972bf1bd4d00a12ef34d300d3dd7 is 1080, key is row0022/info:/1732491389525/Put/seqid=0
2024-11-24T23:36:41,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741849_1025 (size=12509)
2024-11-24T23:36:41,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741849_1025 (size=12509)
2024-11-24T23:36:41,589 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/321e972bf1bd4d00a12ef34d300d3dd7
2024-11-24T23:36:41,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/321e972bf1bd4d00a12ef34d300d3dd7 as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/321e972bf1bd4d00a12ef34d300d3dd7
2024-11-24T23:36:41,610 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/321e972bf1bd4d00a12ef34d300d3dd7, entries=7, sequenceid=42, filesize=12.2 K
2024-11-24T23:36:41,611 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 286f94c7df1b11f1e41ec361e8d9e2ab in 44ms, sequenceid=42, compaction requested=false
2024-11-24T23:36:41,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 286f94c7df1b11f1e41ec361e8d9e2ab:

2024-11-24T23:36:41,611 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-24T23:36:41,611 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:36:41,611 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/ed78c6dadcc240938641a6feb008238f because midkey is the same as first or last row
2024-11-24T23:36:41,902 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T23:36:47,384 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 286f94c7df1b11f1e41ec361e8d9e2ab, had cached 0 bytes from a total of 40219
2024-11-24T23:36:49,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T23:36:49,593 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:36:49,594 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:36:49,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:49,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:49,604 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T23:36:49,604 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T23:36:49,604 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=523817283, stopped=false
2024-11-24T23:36:49,605 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ba7f32532130,40669,1732491312763
2024-11-24T23:36:49,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:36:49,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:36:49,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:49,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:49,676 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:36:49,677 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:36:49,677 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:36:49,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:49,678 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:36:49,678 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:36:49,679 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba7f32532130,45543,1732491313562' *****
2024-11-24T23:36:49,679 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T23:36:49,680 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T23:36:49,681 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T23:36:49,681 INFO  [RS:0;ba7f32532130:45543 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T23:36:49,682 INFO  [RS:0;ba7f32532130:45543 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T23:36:49,682 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(3091): Received CLOSE for 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:36:49,683 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(959): stopping server ba7f32532130,45543,1732491313562
2024-11-24T23:36:49,683 INFO  [RS:0;ba7f32532130:45543 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:36:49,683 INFO  [RS:0;ba7f32532130:45543 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ba7f32532130:45543.
2024-11-24T23:36:49,683 DEBUG [RS:0;ba7f32532130:45543 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:36:49,683 DEBUG [RS:0;ba7f32532130:45543 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:49,684 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T23:36:49,683 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 286f94c7df1b11f1e41ec361e8d9e2ab, disabling compactions & flushes
2024-11-24T23:36:49,684 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T23:36:49,684 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T23:36:49,684 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:36:49,684 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:36:49,684 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T23:36:49,684 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab. after waiting 0 ms
2024-11-24T23:36:49,684 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:36:49,684 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 286f94c7df1b11f1e41ec361e8d9e2ab 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-24T23:36:49,684 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-24T23:36:49,684 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:36:49,684 DEBUG [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 286f94c7df1b11f1e41ec361e8d9e2ab=TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.}
2024-11-24T23:36:49,684 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:36:49,684 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:36:49,684 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:36:49,684 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:36:49,685 DEBUG [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 286f94c7df1b11f1e41ec361e8d9e2ab
2024-11-24T23:36:49,685 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-24T23:36:49,690 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/ca84c048712740dfa018c4db5b6ff8f5 is 1080, key is row0029/info:/1732491403572/Put/seqid=0
2024-11-24T23:36:49,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741850_1026 (size=8193)
2024-11-24T23:36:49,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741850_1026 (size=8193)
2024-11-24T23:36:49,701 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/ca84c048712740dfa018c4db5b6ff8f5
2024-11-24T23:36:49,706 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/.tmp/info/82303974e4054362a9dc71e60d54e989 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab./info:regioninfo/1732491317413/Put/seqid=0
2024-11-24T23:36:49,713 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/.tmp/info/ca84c048712740dfa018c4db5b6ff8f5 as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/ca84c048712740dfa018c4db5b6ff8f5
2024-11-24T23:36:49,715 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741851_1027 (size=7016)
2024-11-24T23:36:49,715 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741851_1027 (size=7016)
2024-11-24T23:36:49,716 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/.tmp/info/82303974e4054362a9dc71e60d54e989
2024-11-24T23:36:49,724 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/ca84c048712740dfa018c4db5b6ff8f5, entries=3, sequenceid=48, filesize=8.0 K
2024-11-24T23:36:49,725 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 286f94c7df1b11f1e41ec361e8d9e2ab in 41ms, sequenceid=48, compaction requested=true
2024-11-24T23:36:49,726 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/a5b0f38fffd8487f814c607e23a184ca, hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/5bf1045fec8643e3a9671fe69c7082ff, hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/208df59c54bc4b318456b191fbab8fe1] to archive
2024-11-24T23:36:49,730 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T23:36:49,734 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/a5b0f38fffd8487f814c607e23a184ca to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/archive/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/a5b0f38fffd8487f814c607e23a184ca
2024-11-24T23:36:49,737 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/5bf1045fec8643e3a9671fe69c7082ff to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/archive/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/5bf1045fec8643e3a9671fe69c7082ff
2024-11-24T23:36:49,739 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/208df59c54bc4b318456b191fbab8fe1 to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/archive/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/info/208df59c54bc4b318456b191fbab8fe1
2024-11-24T23:36:49,746 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/.tmp/ns/ea6c731fd10645c7b79df4cda25abe67 is 43, key is default/ns:d/1732491316705/Put/seqid=0
2024-11-24T23:36:49,753 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741852_1028 (size=5153)
2024-11-24T23:36:49,753 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741852_1028 (size=5153)
2024-11-24T23:36:49,751 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ba7f32532130:40669 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-24T23:36:49,755 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [a5b0f38fffd8487f814c607e23a184ca=12509, 5bf1045fec8643e3a9671fe69c7082ff=12509, 208df59c54bc4b318456b191fbab8fe1=12509]
2024-11-24T23:36:49,756 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/.tmp/ns/ea6c731fd10645c7b79df4cda25abe67
2024-11-24T23:36:49,761 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/default/TestLogRolling-testSlowSyncLogRolling/286f94c7df1b11f1e41ec361e8d9e2ab/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-24T23:36:49,763 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:36:49,764 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 286f94c7df1b11f1e41ec361e8d9e2ab:
Waiting for close lock at 1732491409683Running coprocessor pre-close hooks at 1732491409683Disabling compacts and flushes for region at 1732491409683Disabling writes for close at 1732491409684 (+1 ms)Obtaining lock to block concurrent updates at 1732491409684Preparing flush snapshotting stores in 286f94c7df1b11f1e41ec361e8d9e2ab at 1732491409684Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1732491409684Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab. at 1732491409685 (+1 ms)Flushing 286f94c7df1b11f1e41ec361e8d9e2ab/info: creating writer at 1732491409685Flushing 286f94c7df1b11f1e41ec361e8d9e2ab/info: appending metadata at 1732491409690 (+5 ms)Flushing 286f94c7df1b11f1e41ec361e8d9e2ab/info: closing flushed file at 1732491409690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bdd72e1: reopening flushed file at 1732491409711 (+21 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 286f94c7df1b11f1e41ec361e8d9e2ab in 41ms, sequenceid=48, compaction requested=true at 1732491409726 (+15 ms)Writing region close event to WAL at 1732491409756 (+30 ms)Running coprocessor post-close hooks at 1732491409762 (+6 ms)Closed at 1732491409763 (+1 ms)
2024-11-24T23:36:49,764 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1732491316951.286f94c7df1b11f1e41ec361e8d9e2ab.
2024-11-24T23:36:49,780 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/.tmp/table/38b376079d6b48649b841d1ef5bee320 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1732491317435/Put/seqid=0
2024-11-24T23:36:49,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741853_1029 (size=5396)
2024-11-24T23:36:49,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741853_1029 (size=5396)
2024-11-24T23:36:49,809 INFO  [regionserver/ba7f32532130:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:36:49,880 INFO  [regionserver/ba7f32532130:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-24T23:36:49,881 INFO  [regionserver/ba7f32532130:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-24T23:36:49,885 DEBUG [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-24T23:36:50,085 DEBUG [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-24T23:36:50,189 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/.tmp/table/38b376079d6b48649b841d1ef5bee320
2024-11-24T23:36:50,204 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/.tmp/info/82303974e4054362a9dc71e60d54e989 as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/info/82303974e4054362a9dc71e60d54e989
2024-11-24T23:36:50,215 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/info/82303974e4054362a9dc71e60d54e989, entries=10, sequenceid=11, filesize=6.9 K
2024-11-24T23:36:50,217 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/.tmp/ns/ea6c731fd10645c7b79df4cda25abe67 as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/ns/ea6c731fd10645c7b79df4cda25abe67
2024-11-24T23:36:50,227 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/ns/ea6c731fd10645c7b79df4cda25abe67, entries=2, sequenceid=11, filesize=5.0 K
2024-11-24T23:36:50,228 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/.tmp/table/38b376079d6b48649b841d1ef5bee320 as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/table/38b376079d6b48649b841d1ef5bee320
2024-11-24T23:36:50,238 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/table/38b376079d6b48649b841d1ef5bee320, entries=2, sequenceid=11, filesize=5.3 K
2024-11-24T23:36:50,239 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 555ms, sequenceid=11, compaction requested=false
2024-11-24T23:36:50,244 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-24T23:36:50,245 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:36:50,245 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:36:50,245 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491409684Running coprocessor pre-close hooks at 1732491409684Disabling compacts and flushes for region at 1732491409684Disabling writes for close at 1732491409684Obtaining lock to block concurrent updates at 1732491409685 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732491409685Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1732491409685Flushing stores of hbase:meta,,1.1588230740 at 1732491409686 (+1 ms)Flushing 1588230740/info: creating writer at 1732491409686Flushing 1588230740/info: appending metadata at 1732491409706 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732491409706Flushing 1588230740/ns: creating writer at 1732491409725 (+19 ms)Flushing 1588230740/ns: appending metadata at 1732491409745 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732491409745Flushing 1588230740/table: creating writer at 1732491409764 (+19 ms)Flushing 1588230740/table: appending metadata at 1732491409779 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732491409779Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49ede88: reopening flushed file at 1732491410203 (+424 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b86ca51: reopening flushed file at 1732491410216 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7782bf83: reopening flushed file at 1732491410227 (+11 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 555ms, sequenceid=11, compaction requested=false at 1732491410239 (+12 ms)Writing region close event to WAL at 1732491410240 (+1 ms)Running coprocessor post-close hooks at 1732491410245 (+5 ms)Closed at 1732491410245
2024-11-24T23:36:50,246 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T23:36:50,286 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(976): stopping server ba7f32532130,45543,1732491313562; all regions closed.
2024-11-24T23:36:50,288 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,289 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,289 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,289 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,289 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741834_1010 (size=3066)
2024-11-24T23:36:50,295 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741834_1010 (size=3066)
2024-11-24T23:36:50,300 DEBUG [RS:0;ba7f32532130:45543 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/oldWALs
2024-11-24T23:36:50,300 INFO  [RS:0;ba7f32532130:45543 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C45543%2C1732491313562.meta:.meta(num 1732491316456)
2024-11-24T23:36:50,301 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,301 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,302 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,302 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,302 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741847_1023 (size=12695)
2024-11-24T23:36:50,305 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741847_1023 (size=12695)
2024-11-24T23:36:50,310 DEBUG [RS:0;ba7f32532130:45543 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/oldWALs
2024-11-24T23:36:50,310 INFO  [RS:0;ba7f32532130:45543 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C45543%2C1732491313562:(num 1732491389524)
2024-11-24T23:36:50,310 DEBUG [RS:0;ba7f32532130:45543 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:50,310 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:36:50,310 INFO  [RS:0;ba7f32532130:45543 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:36:50,310 INFO  [RS:0;ba7f32532130:45543 {}] hbase.ChoreService(370): Chore service for: regionserver/ba7f32532130:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T23:36:50,311 INFO  [RS:0;ba7f32532130:45543 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:36:50,311 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:36:50,311 INFO  [RS:0;ba7f32532130:45543 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45543
2024-11-24T23:36:50,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:36:50,359 INFO  [RS:0;ba7f32532130:45543 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:36:50,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba7f32532130,45543,1732491313562
2024-11-24T23:36:50,359 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher.
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$364/0x00007f01349041b8@7868b1cd rejected from java.util.concurrent.ThreadPoolExecutor@3d6a30a4[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?]
	at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?]
	at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4]
2024-11-24T23:36:50,370 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba7f32532130,45543,1732491313562]
2024-11-24T23:36:50,379 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba7f32532130,45543,1732491313562 already deleted, retry=false
2024-11-24T23:36:50,380 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba7f32532130,45543,1732491313562 expired; onlineServers=0
2024-11-24T23:36:50,380 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ba7f32532130,40669,1732491312763' *****
2024-11-24T23:36:50,380 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T23:36:50,380 INFO  [M:0;ba7f32532130:40669 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:36:50,380 INFO  [M:0;ba7f32532130:40669 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:36:50,381 DEBUG [M:0;ba7f32532130:40669 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T23:36:50,381 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T23:36:50,381 DEBUG [M:0;ba7f32532130:40669 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T23:36:50,381 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491315616 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491315616,5,FailOnTimeoutGroup]
2024-11-24T23:36:50,381 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491315609 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491315609,5,FailOnTimeoutGroup]
2024-11-24T23:36:50,381 INFO  [M:0;ba7f32532130:40669 {}] hbase.ChoreService(370): Chore service for: master/ba7f32532130:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T23:36:50,382 INFO  [M:0;ba7f32532130:40669 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:36:50,382 DEBUG [M:0;ba7f32532130:40669 {}] master.HMaster(1795): Stopping service threads
2024-11-24T23:36:50,382 INFO  [M:0;ba7f32532130:40669 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T23:36:50,382 INFO  [M:0;ba7f32532130:40669 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:36:50,383 INFO  [M:0;ba7f32532130:40669 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T23:36:50,383 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T23:36:50,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T23:36:50,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:50,391 DEBUG [M:0;ba7f32532130:40669 {}] zookeeper.ZKUtil(347): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T23:36:50,391 WARN  [M:0;ba7f32532130:40669 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T23:36:50,392 INFO  [M:0;ba7f32532130:40669 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/.lastflushedseqids
2024-11-24T23:36:50,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741854_1030 (size=130)
2024-11-24T23:36:50,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741854_1030 (size=130)
2024-11-24T23:36:50,409 INFO  [M:0;ba7f32532130:40669 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T23:36:50,409 INFO  [M:0;ba7f32532130:40669 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T23:36:50,409 DEBUG [M:0;ba7f32532130:40669 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:36:50,409 INFO  [M:0;ba7f32532130:40669 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:50,409 DEBUG [M:0;ba7f32532130:40669 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:50,409 DEBUG [M:0;ba7f32532130:40669 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:36:50,409 DEBUG [M:0;ba7f32532130:40669 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:50,409 INFO  [M:0;ba7f32532130:40669 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB
2024-11-24T23:36:50,427 DEBUG [M:0;ba7f32532130:40669 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/53a8510ae810489ea98da07473e7309e is 82, key is hbase:meta,,1/info:regioninfo/1732491316563/Put/seqid=0
2024-11-24T23:36:50,434 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741855_1031 (size=5672)
2024-11-24T23:36:50,434 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741855_1031 (size=5672)
2024-11-24T23:36:50,434 INFO  [M:0;ba7f32532130:40669 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/53a8510ae810489ea98da07473e7309e
2024-11-24T23:36:50,459 DEBUG [M:0;ba7f32532130:40669 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/14562f86a24e42fca4bfa1ba9dfcbfbd is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732491317445/Put/seqid=0
2024-11-24T23:36:50,466 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741856_1032 (size=6247)
2024-11-24T23:36:50,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741856_1032 (size=6247)
2024-11-24T23:36:50,467 INFO  [M:0;ba7f32532130:40669 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/14562f86a24e42fca4bfa1ba9dfcbfbd
2024-11-24T23:36:50,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:36:50,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45543-0x1016f3377010001, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:36:50,470 INFO  [RS:0;ba7f32532130:45543 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:36:50,471 INFO  [RS:0;ba7f32532130:45543 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba7f32532130,45543,1732491313562; zookeeper connection closed.
2024-11-24T23:36:50,471 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c972903 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c972903
2024-11-24T23:36:50,472 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T23:36:50,474 INFO  [M:0;ba7f32532130:40669 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 14562f86a24e42fca4bfa1ba9dfcbfbd
2024-11-24T23:36:50,491 DEBUG [M:0;ba7f32532130:40669 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/497f743780564a80890cd2d197feb1d3 is 69, key is ba7f32532130,45543,1732491313562/rs:state/1732491315711/Put/seqid=0
2024-11-24T23:36:50,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741857_1033 (size=5156)
2024-11-24T23:36:50,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741857_1033 (size=5156)
2024-11-24T23:36:50,499 INFO  [M:0;ba7f32532130:40669 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/497f743780564a80890cd2d197feb1d3
2024-11-24T23:36:50,522 DEBUG [M:0;ba7f32532130:40669 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/beaa3b35223d4689a63b9e215072b867 is 52, key is load_balancer_on/state:d/1732491316930/Put/seqid=0
2024-11-24T23:36:50,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741858_1034 (size=5056)
2024-11-24T23:36:50,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741858_1034 (size=5056)
2024-11-24T23:36:50,529 INFO  [M:0;ba7f32532130:40669 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/beaa3b35223d4689a63b9e215072b867
2024-11-24T23:36:50,537 DEBUG [M:0;ba7f32532130:40669 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/53a8510ae810489ea98da07473e7309e as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/53a8510ae810489ea98da07473e7309e
2024-11-24T23:36:50,544 INFO  [M:0;ba7f32532130:40669 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/53a8510ae810489ea98da07473e7309e, entries=8, sequenceid=59, filesize=5.5 K
2024-11-24T23:36:50,545 DEBUG [M:0;ba7f32532130:40669 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/14562f86a24e42fca4bfa1ba9dfcbfbd as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/14562f86a24e42fca4bfa1ba9dfcbfbd
2024-11-24T23:36:50,552 INFO  [M:0;ba7f32532130:40669 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 14562f86a24e42fca4bfa1ba9dfcbfbd
2024-11-24T23:36:50,552 INFO  [M:0;ba7f32532130:40669 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/14562f86a24e42fca4bfa1ba9dfcbfbd, entries=6, sequenceid=59, filesize=6.1 K
2024-11-24T23:36:50,553 DEBUG [M:0;ba7f32532130:40669 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/497f743780564a80890cd2d197feb1d3 as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/497f743780564a80890cd2d197feb1d3
2024-11-24T23:36:50,561 INFO  [M:0;ba7f32532130:40669 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/497f743780564a80890cd2d197feb1d3, entries=1, sequenceid=59, filesize=5.0 K
2024-11-24T23:36:50,562 DEBUG [M:0;ba7f32532130:40669 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/beaa3b35223d4689a63b9e215072b867 as hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/beaa3b35223d4689a63b9e215072b867
2024-11-24T23:36:50,569 INFO  [M:0;ba7f32532130:40669 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/beaa3b35223d4689a63b9e215072b867, entries=1, sequenceid=59, filesize=4.9 K
2024-11-24T23:36:50,570 INFO  [M:0;ba7f32532130:40669 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 161ms, sequenceid=59, compaction requested=false
2024-11-24T23:36:50,572 INFO  [M:0;ba7f32532130:40669 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:50,572 DEBUG [M:0;ba7f32532130:40669 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491410409Disabling compacts and flushes for region at 1732491410409Disabling writes for close at 1732491410409Obtaining lock to block concurrent updates at 1732491410410 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732491410410Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1732491410410Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732491410411 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732491410411Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732491410427 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732491410427Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732491410441 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732491410459 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732491410459Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732491410474 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732491410490 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732491410490Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732491410506 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732491410521 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732491410521Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5db34811: reopening flushed file at 1732491410535 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1080f414: reopening flushed file at 1732491410544 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48e0a866: reopening flushed file at 1732491410552 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43fcfdc8: reopening flushed file at 1732491410561 (+9 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 161ms, sequenceid=59, compaction requested=false at 1732491410570 (+9 ms)Writing region close event to WAL at 1732491410572 (+2 ms)Closed at 1732491410572
2024-11-24T23:36:50,573 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,573 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,574 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,574 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,574 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:50,576 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42031 is added to blk_1073741830_1006 (size=27973)
2024-11-24T23:36:50,577 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44103 is added to blk_1073741830_1006 (size=27973)
2024-11-24T23:36:50,578 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:36:50,578 INFO  [M:0;ba7f32532130:40669 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T23:36:50,578 INFO  [M:0;ba7f32532130:40669 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40669
2024-11-24T23:36:50,578 INFO  [M:0;ba7f32532130:40669 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:36:50,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:36:50,690 INFO  [M:0;ba7f32532130:40669 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:36:50,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40669-0x1016f3377010000, quorum=127.0.0.1:63184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:36:50,695 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:36:50,697 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:36:50,697 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:36:50,697 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:36:50,697 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/hadoop.log.dir/,STOPPED}
2024-11-24T23:36:50,700 WARN  [BP-77390680-172.17.0.3-1732491307723 heartbeating to localhost/127.0.0.1:43777 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:36:50,700 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:36:50,700 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:36:50,700 WARN  [BP-77390680-172.17.0.3-1732491307723 heartbeating to localhost/127.0.0.1:43777 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-77390680-172.17.0.3-1732491307723 (Datanode Uuid f7742adb-ba43-44ed-ad66-7617d5b302f9) service to localhost/127.0.0.1:43777
2024-11-24T23:36:50,702 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f/data/data3/current/BP-77390680-172.17.0.3-1732491307723 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:36:50,702 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f/data/data4/current/BP-77390680-172.17.0.3-1732491307723 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:36:50,703 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:36:50,705 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:36:50,705 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:36:50,705 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:36:50,705 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:36:50,705 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/hadoop.log.dir/,STOPPED}
2024-11-24T23:36:50,707 WARN  [BP-77390680-172.17.0.3-1732491307723 heartbeating to localhost/127.0.0.1:43777 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:36:50,707 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:36:50,707 WARN  [BP-77390680-172.17.0.3-1732491307723 heartbeating to localhost/127.0.0.1:43777 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-77390680-172.17.0.3-1732491307723 (Datanode Uuid 2be3d9de-b3da-4c26-917b-6177c25a15a9) service to localhost/127.0.0.1:43777
2024-11-24T23:36:50,707 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:36:50,708 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f/data/data1/current/BP-77390680-172.17.0.3-1732491307723 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:36:50,708 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/cluster_bc91dd7f-499c-e09d-0e8e-35db4ab9cc8f/data/data2/current/BP-77390680-172.17.0.3-1732491307723 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:36:50,708 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:36:50,717 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:36:50,718 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:36:50,718 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:36:50,718 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:36:50,719 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/hadoop.log.dir/,STOPPED}
2024-11-24T23:36:50,728 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T23:36:50,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T23:36:50,768 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=81 (was 12)
Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43777
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43777 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43777 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@2977aefe
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: master/ba7f32532130:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ForkJoinPool-2-worker-8
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: master/ba7f32532130:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43777
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: regionserver/ba7f32532130:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43777
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43777
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:43777
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43777 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=405 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=233 (was 397), ProcessCount=11 (was 11), AvailableMemoryMB=2788 (was 3420)
2024-11-24T23:36:50,775 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=82, OpenFileDescriptor=405, MaxFileDescriptor=1048576, SystemLoadAverage=233, ProcessCount=11, AvailableMemoryMB=2787
2024-11-24T23:36:50,775 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T23:36:50,775 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/hadoop.log.dir so I do NOT create it in target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee
2024-11-24T23:36:50,775 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d092ee74-1316-b06f-192a-06482656cf09/hadoop.tmp.dir so I do NOT create it in target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee
2024-11-24T23:36:50,775 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2, deleteOnExit=true
2024-11-24T23:36:50,775 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T23:36:50,776 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/test.cache.data in system properties and HBase conf
2024-11-24T23:36:50,776 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T23:36:50,776 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/hadoop.log.dir in system properties and HBase conf
2024-11-24T23:36:50,776 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T23:36:50,776 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T23:36:50,776 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T23:36:50,776 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T23:36:50,776 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/nfs.dump.dir in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/java.io.tmpdir in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:36:50,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T23:36:50,778 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T23:36:50,791 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:36:51,148 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:36:51,155 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:36:51,156 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:36:51,157 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:36:51,157 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:36:51,157 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:36:51,158 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f841e9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:36:51,158 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30a1c2a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:36:51,255 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d95bc23{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/java.io.tmpdir/jetty-localhost-33753-hadoop-hdfs-3_4_1-tests_jar-_-any-7337501702919451720/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:36:51,256 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78fa6004{HTTP/1.1, (http/1.1)}{localhost:33753}
2024-11-24T23:36:51,256 INFO  [Time-limited test {}] server.Server(415): Started @105839ms
2024-11-24T23:36:51,268 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:36:51,552 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:36:51,556 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:36:51,557 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:36:51,557 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:36:51,557 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:36:51,558 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3e23c0c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:36:51,558 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7517d9e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:36:51,657 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d69c419{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/java.io.tmpdir/jetty-localhost-38031-hadoop-hdfs-3_4_1-tests_jar-_-any-11271364256162766415/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:36:51,657 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2f2378c9{HTTP/1.1, (http/1.1)}{localhost:38031}
2024-11-24T23:36:51,657 INFO  [Time-limited test {}] server.Server(415): Started @106241ms
2024-11-24T23:36:51,659 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:36:51,697 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:36:51,703 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:36:51,704 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:36:51,704 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:36:51,704 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:36:51,704 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ce533a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:36:51,705 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@aab268d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:36:51,799 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75434f63{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/java.io.tmpdir/jetty-localhost-39649-hadoop-hdfs-3_4_1-tests_jar-_-any-1424168401330079905/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:36:51,800 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1a2066f8{HTTP/1.1, (http/1.1)}{localhost:39649}
2024-11-24T23:36:51,800 INFO  [Time-limited test {}] server.Server(415): Started @106384ms
2024-11-24T23:36:51,801 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:36:52,933 WARN  [Thread-452 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2/data/data1/current/BP-1848455139-172.17.0.3-1732491410803/current, will proceed with Du for space computation calculation, 
2024-11-24T23:36:52,933 WARN  [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2/data/data2/current/BP-1848455139-172.17.0.3-1732491410803/current, will proceed with Du for space computation calculation, 
2024-11-24T23:36:52,952 WARN  [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:36:52,955 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x66dd46e6aadaac36 with lease ID 0x1d93d5a6e8e0bb09: Processing first storage report for DS-3b94074b-4795-439e-916e-be274d5d46ca from datanode DatanodeRegistration(127.0.0.1:44251, datanodeUuid=9d6d7cd5-7ecf-40c2-a139-7b692cbda2c6, infoPort=43161, infoSecurePort=0, ipcPort=44709, storageInfo=lv=-57;cid=testClusterID;nsid=1124035300;c=1732491410803)
2024-11-24T23:36:52,955 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x66dd46e6aadaac36 with lease ID 0x1d93d5a6e8e0bb09: from storage DS-3b94074b-4795-439e-916e-be274d5d46ca node DatanodeRegistration(127.0.0.1:44251, datanodeUuid=9d6d7cd5-7ecf-40c2-a139-7b692cbda2c6, infoPort=43161, infoSecurePort=0, ipcPort=44709, storageInfo=lv=-57;cid=testClusterID;nsid=1124035300;c=1732491410803), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:36:52,955 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x66dd46e6aadaac36 with lease ID 0x1d93d5a6e8e0bb09: Processing first storage report for DS-f62b58ee-29e9-4640-9571-594a5adcc205 from datanode DatanodeRegistration(127.0.0.1:44251, datanodeUuid=9d6d7cd5-7ecf-40c2-a139-7b692cbda2c6, infoPort=43161, infoSecurePort=0, ipcPort=44709, storageInfo=lv=-57;cid=testClusterID;nsid=1124035300;c=1732491410803)
2024-11-24T23:36:52,955 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x66dd46e6aadaac36 with lease ID 0x1d93d5a6e8e0bb09: from storage DS-f62b58ee-29e9-4640-9571-594a5adcc205 node DatanodeRegistration(127.0.0.1:44251, datanodeUuid=9d6d7cd5-7ecf-40c2-a139-7b692cbda2c6, infoPort=43161, infoSecurePort=0, ipcPort=44709, storageInfo=lv=-57;cid=testClusterID;nsid=1124035300;c=1732491410803), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:36:53,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:36:53,145 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T23:36:53,148 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-24T23:36:53,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-24T23:36:53,290 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2/data/data3/current/BP-1848455139-172.17.0.3-1732491410803/current, will proceed with Du for space computation calculation, 
2024-11-24T23:36:53,290 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2/data/data4/current/BP-1848455139-172.17.0.3-1732491410803/current, will proceed with Du for space computation calculation, 
2024-11-24T23:36:53,307 WARN  [Thread-440 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:36:53,310 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a416a11448623f4 with lease ID 0x1d93d5a6e8e0bb0a: Processing first storage report for DS-a8911261-db58-4be2-882c-2e728d289f99 from datanode DatanodeRegistration(127.0.0.1:34229, datanodeUuid=f1822512-8bda-4f35-9906-95e17ed9518d, infoPort=46577, infoSecurePort=0, ipcPort=37213, storageInfo=lv=-57;cid=testClusterID;nsid=1124035300;c=1732491410803)
2024-11-24T23:36:53,310 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a416a11448623f4 with lease ID 0x1d93d5a6e8e0bb0a: from storage DS-a8911261-db58-4be2-882c-2e728d289f99 node DatanodeRegistration(127.0.0.1:34229, datanodeUuid=f1822512-8bda-4f35-9906-95e17ed9518d, infoPort=46577, infoSecurePort=0, ipcPort=37213, storageInfo=lv=-57;cid=testClusterID;nsid=1124035300;c=1732491410803), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:36:53,310 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a416a11448623f4 with lease ID 0x1d93d5a6e8e0bb0a: Processing first storage report for DS-e11ad81e-520a-4d45-8c64-6d88d9fe6fcb from datanode DatanodeRegistration(127.0.0.1:34229, datanodeUuid=f1822512-8bda-4f35-9906-95e17ed9518d, infoPort=46577, infoSecurePort=0, ipcPort=37213, storageInfo=lv=-57;cid=testClusterID;nsid=1124035300;c=1732491410803)
2024-11-24T23:36:53,310 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a416a11448623f4 with lease ID 0x1d93d5a6e8e0bb0a: from storage DS-e11ad81e-520a-4d45-8c64-6d88d9fe6fcb node DatanodeRegistration(127.0.0.1:34229, datanodeUuid=f1822512-8bda-4f35-9906-95e17ed9518d, infoPort=46577, infoSecurePort=0, ipcPort=37213, storageInfo=lv=-57;cid=testClusterID;nsid=1124035300;c=1732491410803), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:36:53,347 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee
2024-11-24T23:36:53,350 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2/zookeeper_0, clientPort=60258, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T23:36:53,351 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60258
2024-11-24T23:36:53,352 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:53,353 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:53,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:36:53,365 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:36:53,366 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea with version=8
2024-11-24T23:36:53,366 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/hbase-staging
2024-11-24T23:36:53,369 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:36:53,369 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:53,369 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:53,369 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:36:53,369 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:53,370 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:36:53,370 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T23:36:53,370 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:36:53,371 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39177
2024-11-24T23:36:53,373 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39177 connecting to ZooKeeper ensemble=127.0.0.1:60258
2024-11-24T23:36:53,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:391770x0, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:36:53,505 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39177-0x1016f3503110000 connected
2024-11-24T23:36:53,602 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:53,607 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:53,611 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:36:53,611 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea, hbase.cluster.distributed=false
2024-11-24T23:36:53,613 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:36:53,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39177
2024-11-24T23:36:53,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39177
2024-11-24T23:36:53,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39177
2024-11-24T23:36:53,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39177
2024-11-24T23:36:53,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39177
2024-11-24T23:36:53,634 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:36:53,634 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:53,634 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:53,634 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:36:53,634 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:53,634 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:36:53,635 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T23:36:53,635 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:36:53,635 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39283
2024-11-24T23:36:53,638 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39283 connecting to ZooKeeper ensemble=127.0.0.1:60258
2024-11-24T23:36:53,639 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:53,642 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:53,653 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:392830x0, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:36:53,654 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39283-0x1016f3503110001 connected
2024-11-24T23:36:53,654 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:36:53,654 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T23:36:53,656 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T23:36:53,657 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T23:36:53,659 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:36:53,659 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39283
2024-11-24T23:36:53,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39283
2024-11-24T23:36:53,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39283
2024-11-24T23:36:53,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39283
2024-11-24T23:36:53,661 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39283
2024-11-24T23:36:53,674 DEBUG [M:0;ba7f32532130:39177 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ba7f32532130:39177
2024-11-24T23:36:53,674 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ba7f32532130,39177,1732491413369
2024-11-24T23:36:53,685 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:36:53,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:36:53,685 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ba7f32532130,39177,1732491413369
2024-11-24T23:36:53,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:53,695 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T23:36:53,695 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:53,696 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T23:36:53,696 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ba7f32532130,39177,1732491413369 from backup master directory
2024-11-24T23:36:53,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ba7f32532130,39177,1732491413369
2024-11-24T23:36:53,784 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:36:53,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:36:53,785 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:36:53,785 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ba7f32532130,39177,1732491413369
2024-11-24T23:36:53,795 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/hbase.id] with ID: 41a8498a-8fdb-428d-ad4a-9d33a6a34233
2024-11-24T23:36:53,796 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/.tmp/hbase.id
2024-11-24T23:36:53,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:36:53,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:36:53,803 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/.tmp/hbase.id]:[hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/hbase.id]
2024-11-24T23:36:53,816 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:53,816 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T23:36:53,818 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-24T23:36:53,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:53,843 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:53,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:36:53,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:36:53,851 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:36:53,852 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T23:36:53,852 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:36:53,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:36:53,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:36:53,863 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store
2024-11-24T23:36:53,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:36:53,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:36:53,872 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:36:53,872 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:36:53,872 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:53,872 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:53,872 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:36:53,872 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:53,872 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:53,873 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491413872Disabling compacts and flushes for region at 1732491413872Disabling writes for close at 1732491413872Writing region close event to WAL at 1732491413872Closed at 1732491413872
2024-11-24T23:36:53,874 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/.initializing
2024-11-24T23:36:53,874 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/WALs/ba7f32532130,39177,1732491413369
2024-11-24T23:36:53,877 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C39177%2C1732491413369, suffix=, logDir=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/WALs/ba7f32532130,39177,1732491413369, archiveDir=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/oldWALs, maxLogs=10
2024-11-24T23:36:53,878 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C39177%2C1732491413369.1732491413878
2024-11-24T23:36:53,884 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/WALs/ba7f32532130,39177,1732491413369/ba7f32532130%2C39177%2C1732491413369.1732491413878
2024-11-24T23:36:53,886 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46577:46577),(127.0.0.1/127.0.0.1:43161:43161)]
2024-11-24T23:36:53,888 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:36:53,888 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:36:53,888 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:53,889 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:53,891 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:36:53,893 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T23:36:53,893 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:53,894 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:53,894 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:36:53,896 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T23:36:53,896 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:53,897 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:36:53,897 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:36:53,900 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T23:36:53,900 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:53,901 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:36:53,901 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:36:53,903 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T23:36:53,903 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:53,903 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:36:53,903 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:53,904 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:53,905 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:53,906 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:53,907 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:53,907 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T23:36:53,909 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:53,911 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:36:53,912 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=727323, jitterRate=-0.07516133785247803}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T23:36:53,913 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732491413889Initializing all the Stores at 1732491413890 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491413890Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491413891 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491413891Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491413891Cleaning up temporary data from old regions at 1732491413907 (+16 ms)Region opened successfully at 1732491413913 (+6 ms)
2024-11-24T23:36:53,913 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T23:36:53,918 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3194a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:36:53,919 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T23:36:53,919 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T23:36:53,920 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T23:36:53,920 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T23:36:53,921 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T23:36:53,921 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T23:36:53,921 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T23:36:53,924 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T23:36:53,926 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T23:36:53,937 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T23:36:53,938 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T23:36:53,938 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T23:36:53,948 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T23:36:53,948 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T23:36:53,949 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T23:36:53,958 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T23:36:53,959 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T23:36:53,969 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T23:36:53,971 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T23:36:53,979 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T23:36:53,990 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:36:53,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:36:53,990 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:53,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:53,991 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ba7f32532130,39177,1732491413369, sessionid=0x1016f3503110000, setting cluster-up flag (Was=false)
2024-11-24T23:36:54,011 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:54,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:54,043 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T23:36:54,049 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,39177,1732491413369
2024-11-24T23:36:54,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:54,075 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:54,106 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T23:36:54,110 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,39177,1732491413369
2024-11-24T23:36:54,114 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T23:36:54,117 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T23:36:54,118 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T23:36:54,118 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T23:36:54,118 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ba7f32532130,39177,1732491413369
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T23:36:54,120 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:36:54,120 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:36:54,120 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:36:54,120 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:36:54,120 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ba7f32532130:0, corePoolSize=10, maxPoolSize=10
2024-11-24T23:36:54,121 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,121 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:36:54,121 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,123 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732491444123
2024-11-24T23:36:54,123 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T23:36:54,123 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:36:54,123 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T23:36:54,123 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T23:36:54,123 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T23:36:54,124 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T23:36:54,124 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T23:36:54,124 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T23:36:54,124 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,124 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T23:36:54,124 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T23:36:54,125 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T23:36:54,125 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:54,125 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T23:36:54,126 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T23:36:54,126 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T23:36:54,127 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491414127,5,FailOnTimeoutGroup]
2024-11-24T23:36:54,127 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491414127,5,FailOnTimeoutGroup]
2024-11-24T23:36:54,127 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,127 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T23:36:54,128 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,128 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:36:54,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:36:54,136 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T23:36:54,136 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea
2024-11-24T23:36:54,143 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:36:54,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:36:54,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:36:54,146 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:36:54,148 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:36:54,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:54,148 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:54,148 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:36:54,150 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:36:54,150 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:54,150 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:54,150 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:36:54,152 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:36:54,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:54,152 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:54,152 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:36:54,154 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:36:54,154 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:54,154 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:54,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:36:54,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/1588230740
2024-11-24T23:36:54,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/1588230740
2024-11-24T23:36:54,157 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:36:54,157 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:36:54,158 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:36:54,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:36:54,161 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:36:54,161 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=844276, jitterRate=0.07355326414108276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:36:54,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732491414144Initializing all the Stores at 1732491414145 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491414146 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491414146Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491414146Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491414146Cleaning up temporary data from old regions at 1732491414157 (+11 ms)Region opened successfully at 1732491414163 (+6 ms)
2024-11-24T23:36:54,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:36:54,163 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:36:54,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:36:54,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:36:54,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:36:54,164 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:36:54,164 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(746): ClusterId : 41a8498a-8fdb-428d-ad4a-9d33a6a34233
2024-11-24T23:36:54,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491414163Disabling compacts and flushes for region at 1732491414163Disabling writes for close at 1732491414163Writing region close event to WAL at 1732491414164 (+1 ms)Closed at 1732491414164
2024-11-24T23:36:54,164 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T23:36:54,165 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:36:54,165 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T23:36:54,165 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T23:36:54,167 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:36:54,168 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T23:36:54,170 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T23:36:54,170 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T23:36:54,180 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T23:36:54,181 DEBUG [RS:0;ba7f32532130:39283 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e116948, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:36:54,193 DEBUG [RS:0;ba7f32532130:39283 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ba7f32532130:39283
2024-11-24T23:36:54,193 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T23:36:54,193 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T23:36:54,193 DEBUG [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T23:36:54,194 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba7f32532130,39177,1732491413369 with port=39283, startcode=1732491413634
2024-11-24T23:36:54,194 DEBUG [RS:0;ba7f32532130:39283 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T23:36:54,197 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54467, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T23:36:54,197 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39177 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba7f32532130,39283,1732491413634
2024-11-24T23:36:54,197 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39177 {}] master.ServerManager(517): Registering regionserver=ba7f32532130,39283,1732491413634
2024-11-24T23:36:54,199 DEBUG [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea
2024-11-24T23:36:54,199 DEBUG [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38951
2024-11-24T23:36:54,199 DEBUG [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T23:36:54,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:36:54,211 DEBUG [RS:0;ba7f32532130:39283 {}] zookeeper.ZKUtil(111): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba7f32532130,39283,1732491413634
2024-11-24T23:36:54,211 WARN  [RS:0;ba7f32532130:39283 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:36:54,212 INFO  [RS:0;ba7f32532130:39283 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:36:54,212 DEBUG [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/WALs/ba7f32532130,39283,1732491413634
2024-11-24T23:36:54,212 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba7f32532130,39283,1732491413634]
2024-11-24T23:36:54,216 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T23:36:54,219 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T23:36:54,220 INFO  [RS:0;ba7f32532130:39283 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T23:36:54,220 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,220 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T23:36:54,221 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T23:36:54,221 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,221 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,221 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,221 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,222 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,222 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,222 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:36:54,222 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,222 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,222 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,222 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,222 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,222 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:54,222 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:36:54,222 DEBUG [RS:0;ba7f32532130:39283 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:36:54,223 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,223 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,223 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,223 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,223 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,223 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39283,1732491413634-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:36:54,238 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T23:36:54,238 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39283,1732491413634-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,238 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,238 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.Replication(171): ba7f32532130,39283,1732491413634 started
2024-11-24T23:36:54,252 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,252 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(1482): Serving as ba7f32532130,39283,1732491413634, RpcServer on ba7f32532130/172.17.0.3:39283, sessionid=0x1016f3503110001
2024-11-24T23:36:54,252 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T23:36:54,252 DEBUG [RS:0;ba7f32532130:39283 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba7f32532130,39283,1732491413634
2024-11-24T23:36:54,252 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,39283,1732491413634'
2024-11-24T23:36:54,252 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T23:36:54,253 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T23:36:54,254 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T23:36:54,254 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T23:36:54,254 DEBUG [RS:0;ba7f32532130:39283 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba7f32532130,39283,1732491413634
2024-11-24T23:36:54,254 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,39283,1732491413634'
2024-11-24T23:36:54,254 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T23:36:54,254 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T23:36:54,255 DEBUG [RS:0;ba7f32532130:39283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T23:36:54,255 INFO  [RS:0;ba7f32532130:39283 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T23:36:54,255 INFO  [RS:0;ba7f32532130:39283 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T23:36:54,319 WARN  [ba7f32532130:39177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T23:36:54,361 INFO  [RS:0;ba7f32532130:39283 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C39283%2C1732491413634, suffix=, logDir=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/WALs/ba7f32532130,39283,1732491413634, archiveDir=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/oldWALs, maxLogs=32
2024-11-24T23:36:54,366 INFO  [RS:0;ba7f32532130:39283 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C39283%2C1732491413634.1732491414365
2024-11-24T23:36:54,374 INFO  [RS:0;ba7f32532130:39283 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/WALs/ba7f32532130,39283,1732491413634/ba7f32532130%2C39283%2C1732491413634.1732491414365
2024-11-24T23:36:54,378 DEBUG [RS:0;ba7f32532130:39283 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43161:43161),(127.0.0.1/127.0.0.1:46577:46577)]
2024-11-24T23:36:54,569 DEBUG [ba7f32532130:39177 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T23:36:54,571 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba7f32532130,39283,1732491413634
2024-11-24T23:36:54,576 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,39283,1732491413634, state=OPENING
2024-11-24T23:36:54,622 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T23:36:54,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:54,633 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:54,635 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:36:54,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:36:54,635 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,39283,1732491413634}]
2024-11-24T23:36:54,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:36:54,768 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:36:54,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:36:54,792 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T23:36:54,794 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36879, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T23:36:54,799 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T23:36:54,799 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:36:54,802 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C39283%2C1732491413634.meta, suffix=.meta, logDir=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/WALs/ba7f32532130,39283,1732491413634, archiveDir=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/oldWALs, maxLogs=32
2024-11-24T23:36:54,804 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C39283%2C1732491413634.meta.1732491414804.meta
2024-11-24T23:36:54,811 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/WALs/ba7f32532130,39283,1732491413634/ba7f32532130%2C39283%2C1732491413634.meta.1732491414804.meta
2024-11-24T23:36:54,812 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43161:43161),(127.0.0.1/127.0.0.1:46577:46577)]
2024-11-24T23:36:54,813 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:36:54,814 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T23:36:54,814 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T23:36:54,814 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T23:36:54,814 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T23:36:54,814 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:36:54,814 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T23:36:54,814 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T23:36:54,816 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:36:54,817 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:36:54,817 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:54,818 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:54,818 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:36:54,819 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:36:54,820 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:54,820 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:54,820 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:36:54,821 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:36:54,822 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:54,822 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:54,822 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:36:54,823 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:36:54,823 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:54,824 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:54,824 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:36:54,825 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/1588230740
2024-11-24T23:36:54,826 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/1588230740
2024-11-24T23:36:54,828 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:36:54,828 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:36:54,828 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:36:54,830 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:36:54,831 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=713957, jitterRate=-0.09215711057186127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:36:54,831 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T23:36:54,832 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732491414814Writing region info on filesystem at 1732491414814Initializing all the Stores at 1732491414815 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491414815Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491414816 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491414816Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491414816Cleaning up temporary data from old regions at 1732491414828 (+12 ms)Running coprocessor post-open hooks at 1732491414831 (+3 ms)Region opened successfully at 1732491414832 (+1 ms)
2024-11-24T23:36:54,833 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732491414792
2024-11-24T23:36:54,836 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T23:36:54,836 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T23:36:54,837 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,39283,1732491413634
2024-11-24T23:36:54,839 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,39283,1732491413634, state=OPEN
2024-11-24T23:36:54,874 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:36:54,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:36:54,875 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba7f32532130,39283,1732491413634
2024-11-24T23:36:54,875 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:36:54,875 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:36:54,879 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T23:36:54,879 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,39283,1732491413634 in 240 msec
2024-11-24T23:36:54,883 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T23:36:54,883 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 714 msec
2024-11-24T23:36:54,884 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:36:54,884 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T23:36:54,886 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:36:54,887 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,39283,1732491413634, seqNum=-1]
2024-11-24T23:36:54,887 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:36:54,889 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40799, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:36:54,897 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 779 msec
2024-11-24T23:36:54,897 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732491414897, completionTime=-1
2024-11-24T23:36:54,897 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T23:36:54,897 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T23:36:54,900 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T23:36:54,900 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732491474900
2024-11-24T23:36:54,900 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732491534900
2024-11-24T23:36:54,900 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-24T23:36:54,900 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39177,1732491413369-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,900 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39177,1732491413369-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,900 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39177,1732491413369-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,901 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ba7f32532130:39177, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,901 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,901 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,903 DEBUG [master/ba7f32532130:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T23:36:54,906 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.121sec
2024-11-24T23:36:54,907 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T23:36:54,907 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T23:36:54,907 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T23:36:54,907 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T23:36:54,907 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T23:36:54,907 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39177,1732491413369-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:36:54,907 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39177,1732491413369-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T23:36:54,910 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T23:36:54,910 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T23:36:54,910 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39177,1732491413369-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:54,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aadf114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:36:54,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ba7f32532130,39177,-1 for getting cluster id
2024-11-24T23:36:54,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T23:36:54,967 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '41a8498a-8fdb-428d-ad4a-9d33a6a34233'
2024-11-24T23:36:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T23:36:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "41a8498a-8fdb-428d-ad4a-9d33a6a34233"

2024-11-24T23:36:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@408353f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:36:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ba7f32532130,39177,-1]
2024-11-24T23:36:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T23:36:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:54,970 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T23:36:54,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510cc70d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:36:54,972 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:36:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,39283,1732491413634, seqNum=-1]
2024-11-24T23:36:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:36:54,975 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:36:54,977 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ba7f32532130,39177,1732491413369
2024-11-24T23:36:54,978 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:54,981 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T23:36:54,981 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T23:36:54,981 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:36:54,981 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:36:54,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:54,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:54,982 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T23:36:54,982 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T23:36:54,982 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=11609354, stopped=false
2024-11-24T23:36:54,982 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ba7f32532130,39177,1732491413369
2024-11-24T23:36:55,000 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:36:55,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:36:55,001 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:55,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:55,001 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:36:55,001 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:36:55,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:36:55,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:36:55,001 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:36:55,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:55,002 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba7f32532130,39283,1732491413634' *****
2024-11-24T23:36:55,002 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T23:36:55,002 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T23:36:55,002 INFO  [RS:0;ba7f32532130:39283 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T23:36:55,002 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T23:36:55,002 INFO  [RS:0;ba7f32532130:39283 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T23:36:55,003 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(959): stopping server ba7f32532130,39283,1732491413634
2024-11-24T23:36:55,003 INFO  [RS:0;ba7f32532130:39283 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:36:55,003 INFO  [RS:0;ba7f32532130:39283 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ba7f32532130:39283.
2024-11-24T23:36:55,003 DEBUG [RS:0;ba7f32532130:39283 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:36:55,003 DEBUG [RS:0;ba7f32532130:39283 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:55,003 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T23:36:55,003 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T23:36:55,003 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T23:36:55,003 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T23:36:55,004 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-24T23:36:55,004 DEBUG [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-24T23:36:55,004 DEBUG [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-24T23:36:55,004 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:36:55,004 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:36:55,004 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:36:55,004 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:36:55,004 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:36:55,004 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-24T23:36:55,021 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/1588230740/.tmp/ns/9a96960ab6314c10822f48e983077966 is 43, key is default/ns:d/1732491414890/Put/seqid=0
2024-11-24T23:36:55,027 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741835_1011 (size=5153)
2024-11-24T23:36:55,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741835_1011 (size=5153)
2024-11-24T23:36:55,028 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/1588230740/.tmp/ns/9a96960ab6314c10822f48e983077966
2024-11-24T23:36:55,037 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/1588230740/.tmp/ns/9a96960ab6314c10822f48e983077966 as hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/1588230740/ns/9a96960ab6314c10822f48e983077966
2024-11-24T23:36:55,047 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/1588230740/ns/9a96960ab6314c10822f48e983077966, entries=2, sequenceid=6, filesize=5.0 K
2024-11-24T23:36:55,048 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 44ms, sequenceid=6, compaction requested=false
2024-11-24T23:36:55,048 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-24T23:36:55,055 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-24T23:36:55,055 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:36:55,056 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:36:55,056 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491415004Running coprocessor pre-close hooks at 1732491415004Disabling compacts and flushes for region at 1732491415004Disabling writes for close at 1732491415004Obtaining lock to block concurrent updates at 1732491415004Preparing flush snapshotting stores in 1588230740 at 1732491415004Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732491415005 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732491415005Flushing 1588230740/ns: creating writer at 1732491415006 (+1 ms)Flushing 1588230740/ns: appending metadata at 1732491415021 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732491415021Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39be6a9: reopening flushed file at 1732491415036 (+15 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 44ms, sequenceid=6, compaction requested=false at 1732491415048 (+12 ms)Writing region close event to WAL at 1732491415050 (+2 ms)Running coprocessor post-close hooks at 1732491415055 (+5 ms)Closed at 1732491415055
2024-11-24T23:36:55,056 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T23:36:55,204 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(976): stopping server ba7f32532130,39283,1732491413634; all regions closed.
2024-11-24T23:36:55,205 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,205 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,205 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,205 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,206 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,209 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741834_1010 (size=1152)
2024-11-24T23:36:55,209 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741834_1010 (size=1152)
2024-11-24T23:36:55,213 DEBUG [RS:0;ba7f32532130:39283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/oldWALs
2024-11-24T23:36:55,213 INFO  [RS:0;ba7f32532130:39283 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C39283%2C1732491413634.meta:.meta(num 1732491414804)
2024-11-24T23:36:55,214 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,214 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,214 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,214 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,214 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741833_1009 (size=93)
2024-11-24T23:36:55,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741833_1009 (size=93)
2024-11-24T23:36:55,219 DEBUG [RS:0;ba7f32532130:39283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/oldWALs
2024-11-24T23:36:55,219 INFO  [RS:0;ba7f32532130:39283 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C39283%2C1732491413634:(num 1732491414365)
2024-11-24T23:36:55,219 DEBUG [RS:0;ba7f32532130:39283 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:55,219 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:36:55,219 INFO  [RS:0;ba7f32532130:39283 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:36:55,219 INFO  [RS:0;ba7f32532130:39283 {}] hbase.ChoreService(370): Chore service for: regionserver/ba7f32532130:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T23:36:55,220 INFO  [RS:0;ba7f32532130:39283 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:36:55,220 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:36:55,220 INFO  [RS:0;ba7f32532130:39283 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39283
2024-11-24T23:36:55,232 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba7f32532130,39283,1732491413634
2024-11-24T23:36:55,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:36:55,232 INFO  [RS:0;ba7f32532130:39283 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:36:55,242 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba7f32532130,39283,1732491413634]
2024-11-24T23:36:55,253 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba7f32532130,39283,1732491413634 already deleted, retry=false
2024-11-24T23:36:55,253 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba7f32532130,39283,1732491413634 expired; onlineServers=0
2024-11-24T23:36:55,253 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ba7f32532130,39177,1732491413369' *****
2024-11-24T23:36:55,253 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T23:36:55,253 INFO  [M:0;ba7f32532130:39177 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:36:55,253 INFO  [M:0;ba7f32532130:39177 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:36:55,253 DEBUG [M:0;ba7f32532130:39177 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T23:36:55,253 DEBUG [M:0;ba7f32532130:39177 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T23:36:55,253 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T23:36:55,253 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491414127 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491414127,5,FailOnTimeoutGroup]
2024-11-24T23:36:55,253 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491414127 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491414127,5,FailOnTimeoutGroup]
2024-11-24T23:36:55,254 INFO  [M:0;ba7f32532130:39177 {}] hbase.ChoreService(370): Chore service for: master/ba7f32532130:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T23:36:55,254 INFO  [M:0;ba7f32532130:39177 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:36:55,254 DEBUG [M:0;ba7f32532130:39177 {}] master.HMaster(1795): Stopping service threads
2024-11-24T23:36:55,254 INFO  [M:0;ba7f32532130:39177 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T23:36:55,254 INFO  [M:0;ba7f32532130:39177 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:36:55,254 INFO  [M:0;ba7f32532130:39177 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T23:36:55,254 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T23:36:55,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T23:36:55,264 DEBUG [M:0;ba7f32532130:39177 {}] zookeeper.ZKUtil(347): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T23:36:55,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:55,264 WARN  [M:0;ba7f32532130:39177 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T23:36:55,264 INFO  [M:0;ba7f32532130:39177 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/.lastflushedseqids
2024-11-24T23:36:55,271 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741836_1012 (size=99)
2024-11-24T23:36:55,271 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741836_1012 (size=99)
2024-11-24T23:36:55,271 INFO  [M:0;ba7f32532130:39177 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T23:36:55,271 INFO  [M:0;ba7f32532130:39177 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T23:36:55,272 DEBUG [M:0;ba7f32532130:39177 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:36:55,272 INFO  [M:0;ba7f32532130:39177 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:55,272 DEBUG [M:0;ba7f32532130:39177 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:55,272 DEBUG [M:0;ba7f32532130:39177 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:36:55,272 DEBUG [M:0;ba7f32532130:39177 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:55,272 INFO  [M:0;ba7f32532130:39177 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-24T23:36:55,286 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T23:36:55,288 DEBUG [M:0;ba7f32532130:39177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c40747d927714a1da799f4abfbb9e568 is 82, key is hbase:meta,,1/info:regioninfo/1732491414837/Put/seqid=0
2024-11-24T23:36:55,295 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741837_1013 (size=5672)
2024-11-24T23:36:55,295 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:36:55,295 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741837_1013 (size=5672)
2024-11-24T23:36:55,295 INFO  [M:0;ba7f32532130:39177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c40747d927714a1da799f4abfbb9e568
2024-11-24T23:36:55,315 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:36:55,322 DEBUG [M:0;ba7f32532130:39177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f5474ebfc4724c6480eb582128bdb1d6 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732491414896/Put/seqid=0
2024-11-24T23:36:55,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741838_1014 (size=5275)
2024-11-24T23:36:55,329 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741838_1014 (size=5275)
2024-11-24T23:36:55,342 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:36:55,342 INFO  [RS:0;ba7f32532130:39283 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:36:55,343 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39283-0x1016f3503110001, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:36:55,343 INFO  [RS:0;ba7f32532130:39283 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba7f32532130,39283,1732491413634; zookeeper connection closed.
2024-11-24T23:36:55,343 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21bc12e9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21bc12e9
2024-11-24T23:36:55,343 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T23:36:55,731 INFO  [M:0;ba7f32532130:39177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f5474ebfc4724c6480eb582128bdb1d6
2024-11-24T23:36:55,763 DEBUG [M:0;ba7f32532130:39177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f60fcb39d2e04bd5b1533131c0812d35 is 69, key is ba7f32532130,39283,1732491413634/rs:state/1732491414198/Put/seqid=0
2024-11-24T23:36:55,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741839_1015 (size=5156)
2024-11-24T23:36:55,769 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741839_1015 (size=5156)
2024-11-24T23:36:55,769 INFO  [M:0;ba7f32532130:39177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f60fcb39d2e04bd5b1533131c0812d35
2024-11-24T23:36:55,792 DEBUG [M:0;ba7f32532130:39177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/017f9252c62c4e258db7b4299f684edf is 52, key is load_balancer_on/state:d/1732491414979/Put/seqid=0
2024-11-24T23:36:55,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741840_1016 (size=5056)
2024-11-24T23:36:55,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741840_1016 (size=5056)
2024-11-24T23:36:55,798 INFO  [M:0;ba7f32532130:39177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/017f9252c62c4e258db7b4299f684edf
2024-11-24T23:36:55,805 DEBUG [M:0;ba7f32532130:39177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c40747d927714a1da799f4abfbb9e568 as hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c40747d927714a1da799f4abfbb9e568
2024-11-24T23:36:55,812 INFO  [M:0;ba7f32532130:39177 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c40747d927714a1da799f4abfbb9e568, entries=8, sequenceid=29, filesize=5.5 K
2024-11-24T23:36:55,814 DEBUG [M:0;ba7f32532130:39177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f5474ebfc4724c6480eb582128bdb1d6 as hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f5474ebfc4724c6480eb582128bdb1d6
2024-11-24T23:36:55,821 INFO  [M:0;ba7f32532130:39177 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f5474ebfc4724c6480eb582128bdb1d6, entries=3, sequenceid=29, filesize=5.2 K
2024-11-24T23:36:55,822 DEBUG [M:0;ba7f32532130:39177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f60fcb39d2e04bd5b1533131c0812d35 as hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f60fcb39d2e04bd5b1533131c0812d35
2024-11-24T23:36:55,829 INFO  [M:0;ba7f32532130:39177 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f60fcb39d2e04bd5b1533131c0812d35, entries=1, sequenceid=29, filesize=5.0 K
2024-11-24T23:36:55,831 DEBUG [M:0;ba7f32532130:39177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/017f9252c62c4e258db7b4299f684edf as hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/017f9252c62c4e258db7b4299f684edf
2024-11-24T23:36:55,837 INFO  [M:0;ba7f32532130:39177 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38951/user/jenkins/test-data/42d1f0ef-460d-79c4-c5f2-3284799ed1ea/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/017f9252c62c4e258db7b4299f684edf, entries=1, sequenceid=29, filesize=4.9 K
2024-11-24T23:36:55,839 INFO  [M:0;ba7f32532130:39177 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 567ms, sequenceid=29, compaction requested=false
2024-11-24T23:36:55,840 INFO  [M:0;ba7f32532130:39177 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:55,841 DEBUG [M:0;ba7f32532130:39177 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491415271Disabling compacts and flushes for region at 1732491415271Disabling writes for close at 1732491415272 (+1 ms)Obtaining lock to block concurrent updates at 1732491415272Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732491415272Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732491415272Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732491415273 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732491415273Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732491415288 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732491415288Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732491415302 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732491415322 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732491415322Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732491415744 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732491415762 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732491415763 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732491415775 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732491415791 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732491415791Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a7e285c: reopening flushed file at 1732491415803 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@608a5e20: reopening flushed file at 1732491415813 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24648bd6: reopening flushed file at 1732491415821 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@be85e28: reopening flushed file at 1732491415829 (+8 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 567ms, sequenceid=29, compaction requested=false at 1732491415839 (+10 ms)Writing region close event to WAL at 1732491415840 (+1 ms)Closed at 1732491415840
2024-11-24T23:36:55,841 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,841 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,841 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,841 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,841 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:36:55,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44251 is added to blk_1073741830_1006 (size=10311)
2024-11-24T23:36:55,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34229 is added to blk_1073741830_1006 (size=10311)
2024-11-24T23:36:55,844 INFO  [M:0;ba7f32532130:39177 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T23:36:55,844 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:36:55,844 INFO  [M:0;ba7f32532130:39177 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39177
2024-11-24T23:36:55,845 INFO  [M:0;ba7f32532130:39177 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:36:55,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:36:55,995 INFO  [M:0;ba7f32532130:39177 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:36:55,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39177-0x1016f3503110000, quorum=127.0.0.1:60258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:36:55,998 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75434f63{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:36:55,998 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1a2066f8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:36:55,998 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:36:55,998 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@aab268d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:36:55,998 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ce533a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/hadoop.log.dir/,STOPPED}
2024-11-24T23:36:55,999 WARN  [BP-1848455139-172.17.0.3-1732491410803 heartbeating to localhost/127.0.0.1:38951 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:36:55,999 WARN  [BP-1848455139-172.17.0.3-1732491410803 heartbeating to localhost/127.0.0.1:38951 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1848455139-172.17.0.3-1732491410803 (Datanode Uuid f1822512-8bda-4f35-9906-95e17ed9518d) service to localhost/127.0.0.1:38951
2024-11-24T23:36:56,000 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2/data/data3/current/BP-1848455139-172.17.0.3-1732491410803 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:36:56,000 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2/data/data4/current/BP-1848455139-172.17.0.3-1732491410803 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:36:56,000 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:36:56,000 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:36:56,000 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:36:56,002 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7d69c419{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:36:56,003 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2f2378c9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:36:56,003 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:36:56,003 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7517d9e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:36:56,003 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3e23c0c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/hadoop.log.dir/,STOPPED}
2024-11-24T23:36:56,004 WARN  [BP-1848455139-172.17.0.3-1732491410803 heartbeating to localhost/127.0.0.1:38951 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:36:56,004 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:36:56,004 WARN  [BP-1848455139-172.17.0.3-1732491410803 heartbeating to localhost/127.0.0.1:38951 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1848455139-172.17.0.3-1732491410803 (Datanode Uuid 9d6d7cd5-7ecf-40c2-a139-7b692cbda2c6) service to localhost/127.0.0.1:38951
2024-11-24T23:36:56,004 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:36:56,005 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2/data/data1/current/BP-1848455139-172.17.0.3-1732491410803 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:36:56,005 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/cluster_bd90c4af-8a86-d64f-c71b-75b2456440c2/data/data2/current/BP-1848455139-172.17.0.3-1732491410803 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:36:56,005 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:36:56,012 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7d95bc23{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:36:56,013 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78fa6004{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:36:56,013 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:36:56,013 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30a1c2a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:36:56,013 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f841e9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/hadoop.log.dir/,STOPPED}
2024-11-24T23:36:56,018 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T23:36:56,035 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T23:36:56,035 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T23:36:56,035 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/hadoop.log.dir so I do NOT create it in target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02
2024-11-24T23:36:56,035 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dbd3971a-6b0d-7d92-0669-830d6a2b8dee/hadoop.tmp.dir so I do NOT create it in target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02
2024-11-24T23:36:56,035 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e, deleteOnExit=true
2024-11-24T23:36:56,035 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T23:36:56,035 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/test.cache.data in system properties and HBase conf
2024-11-24T23:36:56,036 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T23:36:56,036 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir in system properties and HBase conf
2024-11-24T23:36:56,036 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T23:36:56,036 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T23:36:56,036 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T23:36:56,036 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T23:36:56,036 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:36:56,036 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:36:56,036 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T23:36:56,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:36:56,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T23:36:56,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T23:36:56,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:36:56,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:36:56,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T23:36:56,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/nfs.dump.dir in system properties and HBase conf
2024-11-24T23:36:56,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/java.io.tmpdir in system properties and HBase conf
2024-11-24T23:36:56,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:36:56,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T23:36:56,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T23:36:56,049 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:36:56,224 INFO  [regionserver/ba7f32532130:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:36:56,389 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:36:56,395 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:36:56,399 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:36:56,399 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:36:56,399 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:36:56,400 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:36:56,400 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3150e6db{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:36:56,401 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d790455{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:36:56,495 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7982676d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/java.io.tmpdir/jetty-localhost-38121-hadoop-hdfs-3_4_1-tests_jar-_-any-167647701420723085/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:36:56,496 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2efbdc75{HTTP/1.1, (http/1.1)}{localhost:38121}
2024-11-24T23:36:56,496 INFO  [Time-limited test {}] server.Server(415): Started @111079ms
2024-11-24T23:36:56,508 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:36:56,770 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:36:56,773 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:36:56,774 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:36:56,774 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:36:56,774 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:36:56,775 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf32f74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:36:56,775 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bb5d847{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:36:56,869 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5538b075{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/java.io.tmpdir/jetty-localhost-45353-hadoop-hdfs-3_4_1-tests_jar-_-any-719948112540499813/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:36:56,869 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3f87a993{HTTP/1.1, (http/1.1)}{localhost:45353}
2024-11-24T23:36:56,869 INFO  [Time-limited test {}] server.Server(415): Started @111453ms
2024-11-24T23:36:56,870 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:36:56,899 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:36:56,903 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:36:56,904 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:36:56,904 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:36:56,904 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:36:56,904 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32403ac6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:36:56,905 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4bb19ef9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:36:56,998 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@272348fe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/java.io.tmpdir/jetty-localhost-42367-hadoop-hdfs-3_4_1-tests_jar-_-any-14802660216211352807/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:36:56,998 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10b53169{HTTP/1.1, (http/1.1)}{localhost:42367}
2024-11-24T23:36:56,998 INFO  [Time-limited test {}] server.Server(415): Started @111582ms
2024-11-24T23:36:56,999 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:36:58,218 WARN  [Thread-673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data1/current/BP-1558194410-172.17.0.3-1732491416060/current, will proceed with Du for space computation calculation, 
2024-11-24T23:36:58,218 WARN  [Thread-674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data2/current/BP-1558194410-172.17.0.3-1732491416060/current, will proceed with Du for space computation calculation, 
2024-11-24T23:36:58,238 WARN  [Thread-637 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:36:58,241 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd06757e9aaa59a95 with lease ID 0xb89acf9f033a83ea: Processing first storage report for DS-9812ca79-818f-453d-be00-5ed5f75553cb from datanode DatanodeRegistration(127.0.0.1:43243, datanodeUuid=43d290ec-9444-4b90-a921-7d3b02213370, infoPort=43287, infoSecurePort=0, ipcPort=45891, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060)
2024-11-24T23:36:58,241 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd06757e9aaa59a95 with lease ID 0xb89acf9f033a83ea: from storage DS-9812ca79-818f-453d-be00-5ed5f75553cb node DatanodeRegistration(127.0.0.1:43243, datanodeUuid=43d290ec-9444-4b90-a921-7d3b02213370, infoPort=43287, infoSecurePort=0, ipcPort=45891, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:36:58,241 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd06757e9aaa59a95 with lease ID 0xb89acf9f033a83ea: Processing first storage report for DS-24943718-3190-48bd-82c1-124eb50ec451 from datanode DatanodeRegistration(127.0.0.1:43243, datanodeUuid=43d290ec-9444-4b90-a921-7d3b02213370, infoPort=43287, infoSecurePort=0, ipcPort=45891, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060)
2024-11-24T23:36:58,241 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd06757e9aaa59a95 with lease ID 0xb89acf9f033a83ea: from storage DS-24943718-3190-48bd-82c1-124eb50ec451 node DatanodeRegistration(127.0.0.1:43243, datanodeUuid=43d290ec-9444-4b90-a921-7d3b02213370, infoPort=43287, infoSecurePort=0, ipcPort=45891, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:36:58,354 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data3/current/BP-1558194410-172.17.0.3-1732491416060/current, will proceed with Du for space computation calculation, 
2024-11-24T23:36:58,355 WARN  [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data4/current/BP-1558194410-172.17.0.3-1732491416060/current, will proceed with Du for space computation calculation, 
2024-11-24T23:36:58,379 WARN  [Thread-660 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:36:58,382 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb576279f4659015b with lease ID 0xb89acf9f033a83eb: Processing first storage report for DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa from datanode DatanodeRegistration(127.0.0.1:42531, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=40465, infoSecurePort=0, ipcPort=40453, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060)
2024-11-24T23:36:58,382 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb576279f4659015b with lease ID 0xb89acf9f033a83eb: from storage DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa node DatanodeRegistration(127.0.0.1:42531, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=40465, infoSecurePort=0, ipcPort=40453, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:36:58,382 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb576279f4659015b with lease ID 0xb89acf9f033a83eb: Processing first storage report for DS-267b4a32-f8a8-4a4a-90f0-53ff8a5ecd0d from datanode DatanodeRegistration(127.0.0.1:42531, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=40465, infoSecurePort=0, ipcPort=40453, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060)
2024-11-24T23:36:58,382 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb576279f4659015b with lease ID 0xb89acf9f033a83eb: from storage DS-267b4a32-f8a8-4a4a-90f0-53ff8a5ecd0d node DatanodeRegistration(127.0.0.1:42531, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=40465, infoSecurePort=0, ipcPort=40453, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:36:58,441 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02
2024-11-24T23:36:58,445 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/zookeeper_0, clientPort=52148, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T23:36:58,447 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52148
2024-11-24T23:36:58,447 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:58,449 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:58,459 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43243 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:36:58,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42531 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:36:58,460 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7 with version=8
2024-11-24T23:36:58,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/hbase-staging
2024-11-24T23:36:58,462 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:36:58,463 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:58,463 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:58,463 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:36:58,463 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:58,463 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:36:58,463 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T23:36:58,463 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:36:58,464 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45207
2024-11-24T23:36:58,466 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45207 connecting to ZooKeeper ensemble=127.0.0.1:52148
2024-11-24T23:36:58,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:452070x0, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:36:58,518 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45207-0x1016f3516f70000 connected
2024-11-24T23:36:58,601 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:58,604 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:58,608 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:36:58,608 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7, hbase.cluster.distributed=false
2024-11-24T23:36:58,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:36:58,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45207
2024-11-24T23:36:58,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45207
2024-11-24T23:36:58,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45207
2024-11-24T23:36:58,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45207
2024-11-24T23:36:58,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45207
2024-11-24T23:36:58,629 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:36:58,629 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:58,629 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:58,629 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:36:58,629 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:58,629 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:36:58,629 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T23:36:58,629 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:36:58,630 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34929
2024-11-24T23:36:58,631 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34929 connecting to ZooKeeper ensemble=127.0.0.1:52148
2024-11-24T23:36:58,632 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:58,634 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:58,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:349290x0, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:36:58,643 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:349290x0, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:36:58,643 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34929-0x1016f3516f70001 connected
2024-11-24T23:36:58,643 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T23:36:58,643 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T23:36:58,644 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T23:36:58,645 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:36:58,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34929
2024-11-24T23:36:58,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34929
2024-11-24T23:36:58,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34929
2024-11-24T23:36:58,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34929
2024-11-24T23:36:58,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34929
2024-11-24T23:36:58,658 DEBUG [M:0;ba7f32532130:45207 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ba7f32532130:45207
2024-11-24T23:36:58,659 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ba7f32532130,45207,1732491418462
2024-11-24T23:36:58,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:36:58,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:36:58,672 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ba7f32532130,45207,1732491418462
2024-11-24T23:36:58,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:58,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T23:36:58,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:58,685 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T23:36:58,687 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ba7f32532130,45207,1732491418462 from backup master directory
2024-11-24T23:36:58,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ba7f32532130,45207,1732491418462
2024-11-24T23:36:58,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:36:58,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:36:58,695 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:36:58,695 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ba7f32532130,45207,1732491418462
2024-11-24T23:36:58,701 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/hbase.id] with ID: ddf9a4c4-3008-4ad1-8f96-7c2891cd88f7
2024-11-24T23:36:58,701 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/.tmp/hbase.id
2024-11-24T23:36:58,707 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43243 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:36:58,708 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42531 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:36:58,708 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/.tmp/hbase.id]:[hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/hbase.id]
2024-11-24T23:36:58,721 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:58,721 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T23:36:58,723 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-24T23:36:58,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:58,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:58,741 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43243 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:36:58,741 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42531 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:36:58,741 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:36:58,742 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T23:36:58,742 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:36:58,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42531 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:36:58,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43243 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:36:58,752 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store
2024-11-24T23:36:58,761 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42531 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:36:58,761 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43243 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:36:58,762 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:36:58,762 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:36:58,762 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:58,762 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:58,762 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:36:58,763 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:58,763 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:36:58,763 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491418762Disabling compacts and flushes for region at 1732491418762Disabling writes for close at 1732491418763 (+1 ms)Writing region close event to WAL at 1732491418763Closed at 1732491418763
2024-11-24T23:36:58,764 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/.initializing
2024-11-24T23:36:58,764 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462
2024-11-24T23:36:58,766 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C45207%2C1732491418462, suffix=, logDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462, archiveDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/oldWALs, maxLogs=10
2024-11-24T23:36:58,767 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45207%2C1732491418462.1732491418767
2024-11-24T23:36:58,772 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462/ba7f32532130%2C45207%2C1732491418462.1732491418767
2024-11-24T23:36:58,773 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40465:40465),(127.0.0.1/127.0.0.1:43287:43287)]
2024-11-24T23:36:58,775 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:36:58,776 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:36:58,776 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:58,776 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:58,777 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:36:58,779 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T23:36:58,779 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:58,780 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:58,780 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:36:58,782 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T23:36:58,782 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:58,782 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:36:58,783 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:36:58,784 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T23:36:58,784 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:58,785 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:36:58,785 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:36:58,787 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T23:36:58,787 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:58,787 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:36:58,788 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:58,788 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:58,789 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:58,791 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:58,791 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:58,791 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T23:36:58,793 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:36:58,795 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:36:58,796 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=795857, jitterRate=0.011984780430793762}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T23:36:58,797 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732491418776Initializing all the Stores at 1732491418777 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491418777Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491418777Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491418777Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491418777Cleaning up temporary data from old regions at 1732491418791 (+14 ms)Region opened successfully at 1732491418797 (+6 ms)
2024-11-24T23:36:58,797 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T23:36:58,801 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ecf5cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:36:58,802 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T23:36:58,802 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T23:36:58,802 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T23:36:58,803 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T23:36:58,803 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T23:36:58,804 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T23:36:58,804 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T23:36:58,806 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T23:36:58,807 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T23:36:58,819 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T23:36:58,819 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T23:36:58,820 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T23:36:58,831 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T23:36:58,832 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T23:36:58,834 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T23:36:58,842 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T23:36:58,844 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T23:36:58,853 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T23:36:58,858 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T23:36:58,906 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T23:36:58,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:36:58,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:36:58,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:58,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:58,919 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ba7f32532130,45207,1732491418462, sessionid=0x1016f3516f70000, setting cluster-up flag (Was=false)
2024-11-24T23:36:58,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:58,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:58,968 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T23:36:58,970 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,45207,1732491418462
2024-11-24T23:36:58,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:58,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:59,021 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T23:36:59,023 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,45207,1732491418462
2024-11-24T23:36:59,024 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T23:36:59,026 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T23:36:59,026 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T23:36:59,026 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T23:36:59,027 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ba7f32532130,45207,1732491418462
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T23:36:59,028 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:36:59,028 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:36:59,028 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:36:59,028 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:36:59,028 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ba7f32532130:0, corePoolSize=10, maxPoolSize=10
2024-11-24T23:36:59,029 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,029 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:36:59,029 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,029 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732491449029
2024-11-24T23:36:59,030 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T23:36:59,030 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T23:36:59,030 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T23:36:59,030 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T23:36:59,030 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T23:36:59,030 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T23:36:59,030 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,030 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T23:36:59,030 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T23:36:59,030 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T23:36:59,030 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:36:59,031 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T23:36:59,031 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T23:36:59,031 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T23:36:59,031 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491419031,5,FailOnTimeoutGroup]
2024-11-24T23:36:59,031 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491419031,5,FailOnTimeoutGroup]
2024-11-24T23:36:59,031 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,031 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T23:36:59,031 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,031 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,032 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:59,032 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T23:36:59,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42531 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:36:59,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43243 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:36:59,040 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T23:36:59,041 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7
2024-11-24T23:36:59,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42531 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:36:59,050 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(746): ClusterId : ddf9a4c4-3008-4ad1-8f96-7c2891cd88f7
2024-11-24T23:36:59,050 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T23:36:59,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43243 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:36:59,051 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:36:59,053 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:36:59,054 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:36:59,054 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:59,055 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:59,055 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:36:59,056 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:36:59,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:59,057 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:59,057 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:36:59,059 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:36:59,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:59,059 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:59,059 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:36:59,061 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:36:59,061 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:59,061 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:59,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:36:59,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740
2024-11-24T23:36:59,063 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740
2024-11-24T23:36:59,064 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T23:36:59,064 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T23:36:59,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:36:59,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:36:59,065 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:36:59,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:36:59,068 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:36:59,069 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=799702, jitterRate=0.016874581575393677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:36:59,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732491419051Initializing all the Stores at 1732491419052 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491419052Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491419052Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491419052Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491419052Cleaning up temporary data from old regions at 1732491419064 (+12 ms)Region opened successfully at 1732491419091 (+27 ms)
2024-11-24T23:36:59,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:36:59,092 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:36:59,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:36:59,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:36:59,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:36:59,093 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:36:59,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491419092Disabling compacts and flushes for region at 1732491419092Disabling writes for close at 1732491419092Writing region close event to WAL at 1732491419093 (+1 ms)Closed at 1732491419093
2024-11-24T23:36:59,093 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T23:36:59,093 DEBUG [RS:0;ba7f32532130:34929 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae0bc1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:36:59,097 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:36:59,097 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T23:36:59,097 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T23:36:59,099 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:36:59,101 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T23:36:59,108 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ba7f32532130:34929
2024-11-24T23:36:59,108 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T23:36:59,108 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T23:36:59,108 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T23:36:59,109 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba7f32532130,45207,1732491418462 with port=34929, startcode=1732491418629
2024-11-24T23:36:59,109 DEBUG [RS:0;ba7f32532130:34929 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T23:36:59,112 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50267, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T23:36:59,112 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45207 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba7f32532130,34929,1732491418629
2024-11-24T23:36:59,113 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45207 {}] master.ServerManager(517): Registering regionserver=ba7f32532130,34929,1732491418629
2024-11-24T23:36:59,115 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7
2024-11-24T23:36:59,115 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46275
2024-11-24T23:36:59,115 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T23:36:59,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:36:59,131 DEBUG [RS:0;ba7f32532130:34929 {}] zookeeper.ZKUtil(111): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba7f32532130,34929,1732491418629
2024-11-24T23:36:59,131 WARN  [RS:0;ba7f32532130:34929 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:36:59,131 INFO  [RS:0;ba7f32532130:34929 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:36:59,131 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba7f32532130,34929,1732491418629]
2024-11-24T23:36:59,131 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629
2024-11-24T23:36:59,135 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T23:36:59,138 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T23:36:59,138 INFO  [RS:0;ba7f32532130:34929 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T23:36:59,138 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,140 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T23:36:59,141 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T23:36:59,141 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,141 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,141 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,141 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,141 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,141 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,141 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:36:59,142 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,142 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,142 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,142 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,142 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,142 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:36:59,142 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:36:59,142 DEBUG [RS:0;ba7f32532130:34929 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:36:59,144 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,144 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,144 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,144 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,144 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,144 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,34929,1732491418629-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:36:59,162 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T23:36:59,163 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,34929,1732491418629-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,163 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,163 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.Replication(171): ba7f32532130,34929,1732491418629 started
2024-11-24T23:36:59,175 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,175 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1482): Serving as ba7f32532130,34929,1732491418629, RpcServer on ba7f32532130/172.17.0.3:34929, sessionid=0x1016f3516f70001
2024-11-24T23:36:59,176 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T23:36:59,176 DEBUG [RS:0;ba7f32532130:34929 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba7f32532130,34929,1732491418629
2024-11-24T23:36:59,176 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,34929,1732491418629'
2024-11-24T23:36:59,176 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T23:36:59,176 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T23:36:59,177 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T23:36:59,177 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T23:36:59,177 DEBUG [RS:0;ba7f32532130:34929 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba7f32532130,34929,1732491418629
2024-11-24T23:36:59,177 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,34929,1732491418629'
2024-11-24T23:36:59,177 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T23:36:59,178 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T23:36:59,178 DEBUG [RS:0;ba7f32532130:34929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T23:36:59,178 INFO  [RS:0;ba7f32532130:34929 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T23:36:59,178 INFO  [RS:0;ba7f32532130:34929 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T23:36:59,252 WARN  [ba7f32532130:45207 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T23:36:59,282 INFO  [RS:0;ba7f32532130:34929 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C34929%2C1732491418629, suffix=, logDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629, archiveDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/oldWALs, maxLogs=32
2024-11-24T23:36:59,283 INFO  [RS:0;ba7f32532130:34929 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C34929%2C1732491418629.1732491419283
2024-11-24T23:36:59,293 INFO  [RS:0;ba7f32532130:34929 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283
2024-11-24T23:36:59,296 DEBUG [RS:0;ba7f32532130:34929 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43287:43287),(127.0.0.1/127.0.0.1:40465:40465)]
2024-11-24T23:36:59,502 DEBUG [ba7f32532130:45207 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T23:36:59,503 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba7f32532130,34929,1732491418629
2024-11-24T23:36:59,508 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,34929,1732491418629, state=OPENING
2024-11-24T23:36:59,558 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T23:36:59,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:59,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:36:59,569 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:36:59,569 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:36:59,569 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:36:59,569 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,34929,1732491418629}]
2024-11-24T23:36:59,723 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T23:36:59,729 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52911, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T23:36:59,735 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T23:36:59,735 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:36:59,737 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C34929%2C1732491418629.meta, suffix=.meta, logDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629, archiveDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/oldWALs, maxLogs=32
2024-11-24T23:36:59,738 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
2024-11-24T23:36:59,744 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
2024-11-24T23:36:59,749 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43287:43287),(127.0.0.1/127.0.0.1:40465:40465)]
2024-11-24T23:36:59,754 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:36:59,754 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T23:36:59,754 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T23:36:59,754 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T23:36:59,754 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T23:36:59,755 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:36:59,755 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T23:36:59,755 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T23:36:59,756 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:36:59,757 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:36:59,758 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:59,758 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:59,758 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:36:59,759 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:36:59,759 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:59,760 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:59,760 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:36:59,761 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:36:59,761 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:59,762 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:59,762 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:36:59,763 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:36:59,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:36:59,763 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:36:59,763 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:36:59,764 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740
2024-11-24T23:36:59,765 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740
2024-11-24T23:36:59,767 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:36:59,767 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:36:59,767 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:36:59,769 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:36:59,770 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=866841, jitterRate=0.10224655270576477}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:36:59,770 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T23:36:59,770 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732491419755Writing region info on filesystem at 1732491419755Initializing all the Stores at 1732491419756 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491419756Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491419756Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491419756Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491419756Cleaning up temporary data from old regions at 1732491419767 (+11 ms)Running coprocessor post-open hooks at 1732491419770 (+3 ms)Region opened successfully at 1732491419770
2024-11-24T23:36:59,772 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732491419723
2024-11-24T23:36:59,774 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T23:36:59,774 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T23:36:59,775 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,34929,1732491418629
2024-11-24T23:36:59,777 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,34929,1732491418629, state=OPEN
2024-11-24T23:36:59,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:36:59,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:36:59,816 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba7f32532130,34929,1732491418629
2024-11-24T23:36:59,816 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:36:59,816 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:36:59,821 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T23:36:59,821 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,34929,1732491418629 in 247 msec
2024-11-24T23:36:59,825 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T23:36:59,825 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 724 msec
2024-11-24T23:36:59,827 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:36:59,827 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T23:36:59,829 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:36:59,829 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,34929,1732491418629, seqNum=-1]
2024-11-24T23:36:59,829 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:36:59,831 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45993, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:36:59,837 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 811 msec
2024-11-24T23:36:59,837 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732491419837, completionTime=-1
2024-11-24T23:36:59,837 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T23:36:59,838 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T23:36:59,839 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T23:36:59,839 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732491479839
2024-11-24T23:36:59,839 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732491539839
2024-11-24T23:36:59,839 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-24T23:36:59,840 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,45207,1732491418462-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,840 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,45207,1732491418462-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,840 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,45207,1732491418462-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,840 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ba7f32532130:45207, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,840 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,840 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,842 DEBUG [master/ba7f32532130:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T23:36:59,844 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.149sec
2024-11-24T23:36:59,844 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T23:36:59,844 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T23:36:59,844 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T23:36:59,844 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T23:36:59,844 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T23:36:59,844 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,45207,1732491418462-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:36:59,844 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,45207,1732491418462-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T23:36:59,847 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T23:36:59,847 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T23:36:59,847 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,45207,1732491418462-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:36:59,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e18edd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:36:59,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ba7f32532130,45207,-1 for getting cluster id
2024-11-24T23:36:59,850 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T23:36:59,852 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddf9a4c4-3008-4ad1-8f96-7c2891cd88f7'
2024-11-24T23:36:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T23:36:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddf9a4c4-3008-4ad1-8f96-7c2891cd88f7"

2024-11-24T23:36:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44eb01bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:36:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ba7f32532130,45207,-1]
2024-11-24T23:36:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T23:36:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:36:59,854 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T23:36:59,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344f4078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:36:59,855 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:36:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,34929,1732491418629, seqNum=-1]
2024-11-24T23:36:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:36:59,858 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:36:59,860 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ba7f32532130,45207,1732491418462
2024-11-24T23:36:59,860 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:59,864 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T23:36:59,886 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:36:59,886 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:59,886 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:59,886 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:36:59,886 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:36:59,886 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:36:59,886 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T23:36:59,886 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:36:59,887 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46117
2024-11-24T23:36:59,889 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46117 connecting to ZooKeeper ensemble=127.0.0.1:52148
2024-11-24T23:36:59,890 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:59,892 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:36:59,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461170x0, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:36:59,964 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46117-0x1016f3516f70002, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-24T23:36:59,964 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46117-0x1016f3516f70002 connected
2024-11-24T23:36:59,964 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-24T23:36:59,965 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T23:36:59,966 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T23:36:59,968 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46117-0x1016f3516f70002, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T23:36:59,972 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46117-0x1016f3516f70002, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:36:59,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46117
2024-11-24T23:36:59,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46117
2024-11-24T23:36:59,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46117
2024-11-24T23:36:59,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46117
2024-11-24T23:36:59,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46117
2024-11-24T23:36:59,980 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer(746): ClusterId : ddf9a4c4-3008-4ad1-8f96-7c2891cd88f7
2024-11-24T23:36:59,980 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T23:37:00,030 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T23:37:00,030 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T23:37:00,055 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T23:37:00,057 DEBUG [RS:1;ba7f32532130:46117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e08ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:37:00,075 DEBUG [RS:1;ba7f32532130:46117 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ba7f32532130:46117
2024-11-24T23:37:00,075 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T23:37:00,075 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T23:37:00,075 DEBUG [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T23:37:00,076 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba7f32532130,45207,1732491418462 with port=46117, startcode=1732491419885
2024-11-24T23:37:00,076 DEBUG [RS:1;ba7f32532130:46117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T23:37:00,078 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38845, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T23:37:00,078 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45207 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba7f32532130,46117,1732491419885
2024-11-24T23:37:00,078 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45207 {}] master.ServerManager(517): Registering regionserver=ba7f32532130,46117,1732491419885
2024-11-24T23:37:00,080 DEBUG [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7
2024-11-24T23:37:00,080 DEBUG [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46275
2024-11-24T23:37:00,080 DEBUG [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T23:37:00,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:37:00,093 DEBUG [RS:1;ba7f32532130:46117 {}] zookeeper.ZKUtil(111): regionserver:46117-0x1016f3516f70002, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba7f32532130,46117,1732491419885
2024-11-24T23:37:00,093 WARN  [RS:1;ba7f32532130:46117 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:37:00,093 INFO  [RS:1;ba7f32532130:46117 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:37:00,093 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba7f32532130,46117,1732491419885]
2024-11-24T23:37:00,093 DEBUG [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885
2024-11-24T23:37:00,098 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T23:37:00,100 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T23:37:00,101 INFO  [RS:1;ba7f32532130:46117 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T23:37:00,101 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:00,101 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T23:37:00,102 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T23:37:00,103 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:00,103 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:00,103 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:00,103 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:00,103 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:00,103 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:00,103 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:37:00,103 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:00,103 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:00,104 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:00,104 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:00,104 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:00,104 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:00,104 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:37:00,104 DEBUG [RS:1;ba7f32532130:46117 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:37:00,104 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:00,104 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:00,105 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:00,105 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:00,105 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:00,105 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,46117,1732491419885-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:37:00,119 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T23:37:00,120 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,46117,1732491419885-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:00,120 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:00,120 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.Replication(171): ba7f32532130,46117,1732491419885 started
2024-11-24T23:37:00,132 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:00,132 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer(1482): Serving as ba7f32532130,46117,1732491419885, RpcServer on ba7f32532130/172.17.0.3:46117, sessionid=0x1016f3516f70002
2024-11-24T23:37:00,132 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T23:37:00,132 DEBUG [RS:1;ba7f32532130:46117 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba7f32532130,46117,1732491419885
2024-11-24T23:37:00,132 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;ba7f32532130:46117,5,FailOnTimeoutGroup]
2024-11-24T23:37:00,132 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,46117,1732491419885'
2024-11-24T23:37:00,132 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T23:37:00,133 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-24T23:37:00,133 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-24T23:37:00,133 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T23:37:00,134 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T23:37:00,134 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T23:37:00,134 DEBUG [RS:1;ba7f32532130:46117 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba7f32532130,46117,1732491419885
2024-11-24T23:37:00,134 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,46117,1732491419885'
2024-11-24T23:37:00,134 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T23:37:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is ba7f32532130,45207,1732491418462
2024-11-24T23:37:00,134 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T23:37:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3ee6ce85
2024-11-24T23:37:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-24T23:37:00,135 DEBUG [RS:1;ba7f32532130:46117 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T23:37:00,135 INFO  [RS:1;ba7f32532130:46117 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T23:37:00,135 INFO  [RS:1;ba7f32532130:46117 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T23:37:00,136 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-24T23:37:00,137 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-24T23:37:00,137 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-24T23:37:00,137 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:37:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-24T23:37:00,140 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-24T23:37:00,141 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:00,141 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-24T23:37:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T23:37:00,142 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-24T23:37:00,150 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42531 is added to blk_1073741835_1011 (size=393)
2024-11-24T23:37:00,150 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43243 is added to blk_1073741835_1011 (size=393)
2024-11-24T23:37:00,152 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 60a2fed0d024369a8ac05ba72076bbc9, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7
2024-11-24T23:37:00,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43243 is added to blk_1073741836_1012 (size=76)
2024-11-24T23:37:00,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42531 is added to blk_1073741836_1012 (size=76)
2024-11-24T23:37:00,159 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:37:00,159 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 60a2fed0d024369a8ac05ba72076bbc9, disabling compactions & flushes
2024-11-24T23:37:00,159 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:00,159 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:00,159 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9. after waiting 0 ms
2024-11-24T23:37:00,159 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:00,159 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:00,159 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 60a2fed0d024369a8ac05ba72076bbc9:
Waiting for close lock at 1732491420159Disabling compacts and flushes for region at 1732491420159Disabling writes for close at 1732491420159Writing region close event to WAL at 1732491420159Closed at 1732491420159
2024-11-24T23:37:00,160 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-24T23:37:00,161 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1732491420160"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732491420160"}]},"ts":"1732491420160"}
2024-11-24T23:37:00,163 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-24T23:37:00,164 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-24T23:37:00,165 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732491420164"}]},"ts":"1732491420164"}
2024-11-24T23:37:00,167 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-24T23:37:00,167 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=60a2fed0d024369a8ac05ba72076bbc9, ASSIGN}]
2024-11-24T23:37:00,169 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=60a2fed0d024369a8ac05ba72076bbc9, ASSIGN
2024-11-24T23:37:00,170 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=60a2fed0d024369a8ac05ba72076bbc9, ASSIGN; state=OFFLINE, location=ba7f32532130,34929,1732491418629; forceNewPlan=false, retain=false
2024-11-24T23:37:00,237 INFO  [RS:1;ba7f32532130:46117 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C46117%2C1732491419885, suffix=, logDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885, archiveDir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/oldWALs, maxLogs=32
2024-11-24T23:37:00,238 INFO  [RS:1;ba7f32532130:46117 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C46117%2C1732491419885.1732491420238
2024-11-24T23:37:00,244 INFO  [RS:1;ba7f32532130:46117 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
2024-11-24T23:37:00,245 DEBUG [RS:1;ba7f32532130:46117 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43287:43287),(127.0.0.1/127.0.0.1:40465:40465)]
2024-11-24T23:37:00,317 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T23:37:00,322 INFO  [ba7f32532130:45207 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-24T23:37:00,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:00,322 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=60a2fed0d024369a8ac05ba72076bbc9, regionState=OPENING, regionLocation=ba7f32532130,34929,1732491418629
2024-11-24T23:37:00,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=60a2fed0d024369a8ac05ba72076bbc9, ASSIGN because future has completed
2024-11-24T23:37:00,326 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a2fed0d024369a8ac05ba72076bbc9, server=ba7f32532130,34929,1732491418629}]
2024-11-24T23:37:00,337 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:00,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:00,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:00,491 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:00,491 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 60a2fed0d024369a8ac05ba72076bbc9, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:37:00,492 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:00,493 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:37:00,493 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:00,493 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:00,496 INFO  [StoreOpener-60a2fed0d024369a8ac05ba72076bbc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 60a2fed0d024369a8ac05ba72076bbc9 
2024-11-24T23:37:00,498 INFO  [StoreOpener-60a2fed0d024369a8ac05ba72076bbc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a2fed0d024369a8ac05ba72076bbc9 columnFamilyName info
2024-11-24T23:37:00,498 DEBUG [StoreOpener-60a2fed0d024369a8ac05ba72076bbc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:00,498 INFO  [StoreOpener-60a2fed0d024369a8ac05ba72076bbc9-1 {}] regionserver.HStore(327): Store=60a2fed0d024369a8ac05ba72076bbc9/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:37:00,499 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:00,499 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:00,500 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:00,500 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:00,500 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:00,502 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:00,504 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:37:00,504 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 60a2fed0d024369a8ac05ba72076bbc9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=706487, jitterRate=-0.101655974984169}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T23:37:00,504 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:00,505 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 60a2fed0d024369a8ac05ba72076bbc9:
Running coprocessor pre-open hook at 1732491420493Writing region info on filesystem at 1732491420493Initializing all the Stores at 1732491420496 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491420496Cleaning up temporary data from old regions at 1732491420500 (+4 ms)Running coprocessor post-open hooks at 1732491420504 (+4 ms)Region opened successfully at 1732491420505 (+1 ms)
2024-11-24T23:37:00,506 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9., pid=6, masterSystemTime=1732491420480
2024-11-24T23:37:00,509 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:00,509 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:00,510 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=60a2fed0d024369a8ac05ba72076bbc9, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,34929,1732491418629
2024-11-24T23:37:00,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a2fed0d024369a8ac05ba72076bbc9, server=ba7f32532130,34929,1732491418629 because future has completed
2024-11-24T23:37:00,518 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-24T23:37:00,518 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 60a2fed0d024369a8ac05ba72076bbc9, server=ba7f32532130,34929,1732491418629 in 189 msec
2024-11-24T23:37:00,521 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-24T23:37:00,521 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=60a2fed0d024369a8ac05ba72076bbc9, ASSIGN in 351 msec
2024-11-24T23:37:00,522 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-24T23:37:00,522 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732491420522"}]},"ts":"1732491420522"}
2024-11-24T23:37:00,524 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-24T23:37:00,526 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-24T23:37:00,528 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 389 msec
2024-11-24T23:37:03,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T23:37:03,144 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-24T23:37:03,147 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-24T23:37:03,148 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-24T23:37:03,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:37:03,149 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-24T23:37:03,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-24T23:37:03,149 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-24T23:37:05,260 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T23:37:05,264 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:05,283 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:05,285 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:05,285 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:05,292 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-24T23:37:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T23:37:10,184 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-24T23:37:10,184 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-24T23:37:10,195 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-24T23:37:10,195 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:10,208 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:37:10,212 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:37:10,212 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:37:10,212 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:37:10,212 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:37:10,213 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30008f24{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:37:10,213 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ab5b96c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:37:10,306 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3fd17220{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/java.io.tmpdir/jetty-localhost-46829-hadoop-hdfs-3_4_1-tests_jar-_-any-7869975101152715361/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:10,307 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c68f920{HTTP/1.1, (http/1.1)}{localhost:46829}
2024-11-24T23:37:10,307 INFO  [Time-limited test {}] server.Server(415): Started @124890ms
2024-11-24T23:37:10,308 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:37:10,337 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:37:10,340 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:37:10,341 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:37:10,341 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:37:10,341 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:37:10,342 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f85c2b2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:37:10,342 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64b89ed5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:37:10,435 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d82aaea{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/java.io.tmpdir/jetty-localhost-35879-hadoop-hdfs-3_4_1-tests_jar-_-any-3385632281816581299/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:10,435 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3904e150{HTTP/1.1, (http/1.1)}{localhost:35879}
2024-11-24T23:37:10,435 INFO  [Time-limited test {}] server.Server(415): Started @125019ms
2024-11-24T23:37:10,436 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:37:10,464 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:37:10,467 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:37:10,468 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:37:10,468 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:37:10,468 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:37:10,468 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b21f544{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:37:10,468 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47432b7b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:37:10,561 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@20faceaa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/java.io.tmpdir/jetty-localhost-41471-hadoop-hdfs-3_4_1-tests_jar-_-any-7381399601945757114/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:10,562 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e1ad43e{HTTP/1.1, (http/1.1)}{localhost:41471}
2024-11-24T23:37:10,562 INFO  [Time-limited test {}] server.Server(415): Started @125146ms
2024-11-24T23:37:10,563 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:37:11,729 WARN  [Thread-869 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data5/current/BP-1558194410-172.17.0.3-1732491416060/current, will proceed with Du for space computation calculation, 
2024-11-24T23:37:11,729 WARN  [Thread-870 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data6/current/BP-1558194410-172.17.0.3-1732491416060/current, will proceed with Du for space computation calculation, 
2024-11-24T23:37:11,751 WARN  [Thread-810 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:37:11,754 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x95b0b71d100b7935 with lease ID 0xb89acf9f033a83ec: Processing first storage report for DS-b015f799-15b8-418e-99d4-9f95a934fe13 from datanode DatanodeRegistration(127.0.0.1:43597, datanodeUuid=231a53f4-5841-4c22-a9e9-68621e91ea2b, infoPort=42465, infoSecurePort=0, ipcPort=33739, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060)
2024-11-24T23:37:11,754 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x95b0b71d100b7935 with lease ID 0xb89acf9f033a83ec: from storage DS-b015f799-15b8-418e-99d4-9f95a934fe13 node DatanodeRegistration(127.0.0.1:43597, datanodeUuid=231a53f4-5841-4c22-a9e9-68621e91ea2b, infoPort=42465, infoSecurePort=0, ipcPort=33739, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T23:37:11,754 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x95b0b71d100b7935 with lease ID 0xb89acf9f033a83ec: Processing first storage report for DS-65b2eb61-9ef2-4914-a2bd-a2c4776265f9 from datanode DatanodeRegistration(127.0.0.1:43597, datanodeUuid=231a53f4-5841-4c22-a9e9-68621e91ea2b, infoPort=42465, infoSecurePort=0, ipcPort=33739, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060)
2024-11-24T23:37:11,754 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x95b0b71d100b7935 with lease ID 0xb89acf9f033a83ec: from storage DS-65b2eb61-9ef2-4914-a2bd-a2c4776265f9 node DatanodeRegistration(127.0.0.1:43597, datanodeUuid=231a53f4-5841-4c22-a9e9-68621e91ea2b, infoPort=42465, infoSecurePort=0, ipcPort=33739, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:37:11,889 WARN  [Thread-882 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9/current/BP-1558194410-172.17.0.3-1732491416060/current, will proceed with Du for space computation calculation, 
2024-11-24T23:37:11,889 WARN  [Thread-883 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10/current/BP-1558194410-172.17.0.3-1732491416060/current, will proceed with Du for space computation calculation, 
2024-11-24T23:37:11,889 WARN  [Thread-884 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data7/current/BP-1558194410-172.17.0.3-1732491416060/current, will proceed with Du for space computation calculation, 
2024-11-24T23:37:11,889 WARN  [Thread-885 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data8/current/BP-1558194410-172.17.0.3-1732491416060/current, will proceed with Du for space computation calculation, 
2024-11-24T23:37:11,907 WARN  [Thread-832 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:37:11,909 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7a1a51f614e6cfe6 with lease ID 0xb89acf9f033a83ed: Processing first storage report for DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969 from datanode DatanodeRegistration(127.0.0.1:35595, datanodeUuid=f44c8ee5-2ace-41cd-be4a-1a6b0e0dea84, infoPort=39143, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060)
2024-11-24T23:37:11,909 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7a1a51f614e6cfe6 with lease ID 0xb89acf9f033a83ed: from storage DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969 node DatanodeRegistration(127.0.0.1:35595, datanodeUuid=f44c8ee5-2ace-41cd-be4a-1a6b0e0dea84, infoPort=39143, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:37:11,910 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7a1a51f614e6cfe6 with lease ID 0xb89acf9f033a83ed: Processing first storage report for DS-d32ca5d8-7ac6-442c-a5b3-c9264019ab50 from datanode DatanodeRegistration(127.0.0.1:35595, datanodeUuid=f44c8ee5-2ace-41cd-be4a-1a6b0e0dea84, infoPort=39143, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060)
2024-11-24T23:37:11,910 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7a1a51f614e6cfe6 with lease ID 0xb89acf9f033a83ed: from storage DS-d32ca5d8-7ac6-442c-a5b3-c9264019ab50 node DatanodeRegistration(127.0.0.1:35595, datanodeUuid=f44c8ee5-2ace-41cd-be4a-1a6b0e0dea84, infoPort=39143, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:37:11,911 WARN  [Thread-854 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:37:11,913 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc3b7ce21b42bd4cf with lease ID 0xb89acf9f033a83ee: Processing first storage report for DS-bd38573c-39c9-41fc-9860-aac18083f853 from datanode DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060)
2024-11-24T23:37:11,913 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc3b7ce21b42bd4cf with lease ID 0xb89acf9f033a83ee: from storage DS-bd38573c-39c9-41fc-9860-aac18083f853 node DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:37:11,913 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc3b7ce21b42bd4cf with lease ID 0xb89acf9f033a83ee: Processing first storage report for DS-2b7920eb-3b4f-4d58-b8fd-a83779e02ef5 from datanode DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060)
2024-11-24T23:37:11,914 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc3b7ce21b42bd4cf with lease ID 0xb89acf9f033a83ee: from storage DS-2b7920eb-3b4f-4d58-b8fd-a83779e02ef5 node DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:37:11,998 WARN  [ResponseProcessor for block BP-1558194410-172.17.0.3-1732491416060:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1558194410-172.17.0.3-1732491416060:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:11,998 WARN  [ResponseProcessor for block BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:11,998 WARN  [ResponseProcessor for block BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:11,998 WARN  [ResponseProcessor for block BP-1558194410-172.17.0.3-1732491416060:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1558194410-172.17.0.3-1732491416060:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1558194410-172.17.0.3-1732491416060:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:11,999 WARN  [DataStreamer for file /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462/ba7f32532130%2C45207%2C1732491418462.1732491418767 block BP-1558194410-172.17.0.3-1732491416060:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:12,000 WARN  [DataStreamer for file /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta block BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK], DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:12,000 WARN  [DataStreamer for file /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238 block BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK], DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:12,000 WARN  [DataStreamer for file /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 block BP-1558194410-172.17.0.3-1732491416060:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK], DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:11,999 WARN  [PacketResponder: BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42531] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:12,000 WARN  [PacketResponder: BP-1558194410-172.17.0.3-1732491416060:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42531] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:12,000 WARN  [PacketResponder: BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42531] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:12,002 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:42968 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43243:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42968 dst: /127.0.0.1:43243
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:12,002 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:42980 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43243:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42980 dst: /127.0.0.1:43243
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:12,001 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1560667721_22 at /127.0.0.1:41572 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42531:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41572 dst: /127.0.0.1:42531
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:12,003 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@272348fe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:12,003 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2060887705_22 at /127.0.0.1:43004 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:43243:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43004 dst: /127.0.0.1:43243
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:12,002 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1560667721_22 at /127.0.0.1:42938 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43243:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42938 dst: /127.0.0.1:43243
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:12,004 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:41600 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42531:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41600 dst: /127.0.0.1:42531
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:12,004 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10b53169{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:37:12,004 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:37:12,004 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2060887705_22 at /127.0.0.1:41620 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:42531:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41620 dst: /127.0.0.1:42531
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:12,004 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:41594 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42531:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41594 dst: /127.0.0.1:42531
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:12,004 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4bb19ef9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:37:12,005 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32403ac6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,STOPPED}
2024-11-24T23:37:12,006 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:37:12,006 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:37:12,006 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1558194410-172.17.0.3-1732491416060 (Datanode Uuid 14ab36d8-5a0d-4818-be52-baede7a0917f) service to localhost/127.0.0.1:46275
2024-11-24T23:37:12,006 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:37:12,007 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data4/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:12,007 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:37:12,007 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data3/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:12,008 WARN  [DataStreamer for file /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238 block BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:12,008 WARN  [DataStreamer for file /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta block BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:12,008 WARN  [DataStreamer for file /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462/ba7f32532130%2C45207%2C1732491418462.1732491418767 block BP-1558194410-172.17.0.3-1732491416060:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:12,008 WARN  [DataStreamer for file /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 block BP-1558194410-172.17.0.3-1732491416060:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:12,009 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5538b075{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:12,010 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3f87a993{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:37:12,010 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:37:12,010 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bb5d847{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:37:12,010 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf32f74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,STOPPED}
2024-11-24T23:37:12,011 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:37:12,011 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:37:12,011 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1558194410-172.17.0.3-1732491416060 (Datanode Uuid 43d290ec-9444-4b90-a921-7d3b02213370) service to localhost/127.0.0.1:46275
2024-11-24T23:37:12,011 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:37:12,011 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data1/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:12,012 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data2/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:12,012 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:37:12,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9., hostname=ba7f32532130,34929,1732491418629, seqNum=2]
2024-11-24T23:37:12,017 ERROR [FSHLog-0-hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7-prefix:ba7f32532130,34929,1732491418629 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:12,017 WARN  [FSHLog-0-hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7-prefix:ba7f32532130,34929,1732491418629 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:12,017 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C34929%2C1732491418629:(num 1732491419283) roll requested
2024-11-24T23:37:12,017 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C34929%2C1732491418629.1732491432017
2024-11-24T23:37:12,023 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:12,023 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:12,023 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:12,023 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:12,023 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:12,024 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491432017
2024-11-24T23:37:12,024 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:12,024 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:12,024 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42465:42465),(127.0.0.1/127.0.0.1:39143:39143)]
2024-11-24T23:37:12,024 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 is not closed yet, will try archiving it next time
2024-11-24T23:37:12,025 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-24T23:37:12,025 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-24T23:37:12,026 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283
2024-11-24T23:37:12,028 WARN  [IPC Server handler 4 on default port 46275 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009
2024-11-24T23:37:12,031 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 after 4ms
2024-11-24T23:37:12,106 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:12,744 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:14,025 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:14,026 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491432017
2024-11-24T23:37:14,027 WARN  [ResponseProcessor for block BP-1558194410-172.17.0.3-1732491416060:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1558194410-172.17.0.3-1732491416060:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:14,027 WARN  [DataStreamer for file /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491432017 block BP-1558194410-172.17.0.3-1732491416060:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK], DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:14,028 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:39154 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:43597:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39154 dst: /127.0.0.1:43597
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:14,028 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50280 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:35595:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50280 dst: /127.0.0.1:35595
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:14,077 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3fd17220{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:14,078 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c68f920{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:37:14,078 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:37:14,078 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ab5b96c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:37:14,079 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30008f24{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,STOPPED}
2024-11-24T23:37:14,081 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:37:14,081 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:37:14,081 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1558194410-172.17.0.3-1732491416060 (Datanode Uuid 231a53f4-5841-4c22-a9e9-68621e91ea2b) service to localhost/127.0.0.1:46275
2024-11-24T23:37:14,081 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:37:14,082 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data5/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:14,082 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data6/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:14,083 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:37:14,106 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:14,745 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:16,026 WARN  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]]
2024-11-24T23:37:16,026 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:16,027 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C34929%2C1732491418629:(num 1732491432017) roll requested
2024-11-24T23:37:16,027 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C34929%2C1732491418629.1732491436027
2024-11-24T23:37:16,033 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 after 4007ms
2024-11-24T23:37:16,035 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43243
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:16,035 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50964 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741839_1021] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10]'}, localName='127.0.0.1:37325', datanodeUuid='bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3', xmitsInProgress=0}:Exception transferring block BP-1558194410-172.17.0.3-1732491416060:blk_1073741839_1021 to mirror 127.0.0.1:43243
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:16,035 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]) is bad.
2024-11-24T23:37:16,035 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741839_1021
2024-11-24T23:37:16,035 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50964 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741839_1021] {}] datanode.BlockReceiver(316): Block 1073741839 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T23:37:16,035 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50964 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:37325:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50964 dst: /127.0.0.1:37325
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:16,038 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]
2024-11-24T23:37:16,042 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:16,042 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK], DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:16,042 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741840_1022
2024-11-24T23:37:16,043 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
2024-11-24T23:37:16,045 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43597
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:16,045 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50292 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741841_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data8]'}, localName='127.0.0.1:35595', datanodeUuid='f44c8ee5-2ace-41cd-be4a-1a6b0e0dea84', xmitsInProgress=0}:Exception transferring block BP-1558194410-172.17.0.3-1732491416060:blk_1073741841_1023 to mirror 127.0.0.1:43597
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:16,045 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:16,045 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741841_1023
2024-11-24T23:37:16,045 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50292 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741841_1023] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T23:37:16,045 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50292 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:35595:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50292 dst: /127.0.0.1:35595
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:16,046 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]
2024-11-24T23:37:16,050 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:16,050 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:16,050 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:16,050 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:16,050 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:16,050 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491432017 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491436027
2024-11-24T23:37:16,051 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44357:44357),(127.0.0.1/127.0.0.1:39143:39143)]
2024-11-24T23:37:16,051 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 is not closed yet, will try archiving it next time
2024-11-24T23:37:16,051 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491432017 is not closed yet, will try archiving it next time
2024-11-24T23:37:16,052 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35595 is added to blk_1073741838_1020 (size=2431)
2024-11-24T23:37:16,090 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-24T23:37:16,107 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:16,453 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 is not closed yet, will try archiving it next time
2024-11-24T23:37:16,746 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:17,920 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@2879aff9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35595, datanodeUuid=f44c8ee5-2ace-41cd-be4a-1a6b0e0dea84, infoPort=39143, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741838_1020 to 127.0.0.1:43597 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:18,052 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,098 WARN  [ResponseProcessor for block BP-1558194410-172.17.0.3-1732491416060:blk_1073741842_1024 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1558194410-172.17.0.3-1732491416060:blk_1073741842_1024
java.io.IOException: Bad response ERROR for BP-1558194410-172.17.0.3-1732491416060:blk_1073741842_1024 from datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,099 WARN  [DataStreamer for file /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491436027 block BP-1558194410-172.17.0.3-1732491416060:blk_1073741842_1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK], DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:18,099 WARN  [PacketResponder: BP-1558194410-172.17.0.3-1732491416060:blk_1073741842_1024, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35595] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:18,100 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50972 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:37325:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50972 dst: /127.0.0.1:37325
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:18,100 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50308 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:35595:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50308 dst: /127.0.0.1:35595
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:18,107 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,129 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d82aaea{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:18,129 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3904e150{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:37:18,129 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:37:18,129 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64b89ed5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:37:18,130 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f85c2b2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,STOPPED}
2024-11-24T23:37:18,131 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:37:18,131 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1558194410-172.17.0.3-1732491416060 (Datanode Uuid f44c8ee5-2ace-41cd-be4a-1a6b0e0dea84) service to localhost/127.0.0.1:46275
2024-11-24T23:37:18,132 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data7/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:18,132 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data8/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:18,132 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:37:18,132 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:37:18,132 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:37:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] regionserver.HRegion(8855): Flush requested on 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:18,143 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 60a2fed0d024369a8ac05ba72076bbc9 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T23:37:18,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/c117af440de54a74bda89c1b6a8ac1b2 is 1080, key is row0002/info:/1732491434085/Put/seqid=0
2024-11-24T23:37:18,171 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,171 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:18,171 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741843_1026
2024-11-24T23:37:18,172 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]
2024-11-24T23:37:18,174 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,174 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK], DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]) is bad.
2024-11-24T23:37:18,174 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741844_1027
2024-11-24T23:37:18,175 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]
2024-11-24T23:37:18,177 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,177 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:18,177 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741845_1028
2024-11-24T23:37:18,178 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:18,185 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42531
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,185 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50998 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10]'}, localName='127.0.0.1:37325', datanodeUuid='bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3', xmitsInProgress=0}:Exception transferring block BP-1558194410-172.17.0.3-1732491416060:blk_1073741846_1029 to mirror 127.0.0.1:42531
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:18,186 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK], DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:18,186 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741846_1029
2024-11-24T23:37:18,186 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50998 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-24T23:37:18,186 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:50998 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:37325:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50998 dst: /127.0.0.1:37325
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:18,187 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
2024-11-24T23:37:18,188 WARN  [IPC Server handler 4 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T23:37:18,188 WARN  [IPC Server handler 4 on default port 46275 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T23:37:18,188 WARN  [IPC Server handler 4 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T23:37:18,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741847_1030 (size=10347)
2024-11-24T23:37:18,593 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/c117af440de54a74bda89c1b6a8ac1b2
2024-11-24T23:37:18,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/c117af440de54a74bda89c1b6a8ac1b2 as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/c117af440de54a74bda89c1b6a8ac1b2
2024-11-24T23:37:18,613 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/c117af440de54a74bda89c1b6a8ac1b2, entries=5, sequenceid=11, filesize=10.1 K
2024-11-24T23:37:18,614 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 60a2fed0d024369a8ac05ba72076bbc9 in 471ms, sequenceid=11, compaction requested=false
2024-11-24T23:37:18,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 60a2fed0d024369a8ac05ba72076bbc9:

2024-11-24T23:37:18,746 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] regionserver.HRegion(8855): Flush requested on 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:18,786 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 60a2fed0d024369a8ac05ba72076bbc9 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-24T23:37:18,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/3b9acc9f10e24100a7547bb0375371ad is 1080, key is row0007/info:/1732491438145/Put/seqid=0
2024-11-24T23:37:18,795 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,795 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK], DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:18,795 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741848_1031
2024-11-24T23:37:18,796 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
2024-11-24T23:37:18,797 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,797 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:18,797 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741849_1032
2024-11-24T23:37:18,798 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:18,799 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,799 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:18,799 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741850_1033
2024-11-24T23:37:18,800 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]
2024-11-24T23:37:18,801 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:18,801 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]) is bad.
2024-11-24T23:37:18,801 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741851_1034
2024-11-24T23:37:18,802 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]
2024-11-24T23:37:18,802 WARN  [IPC Server handler 4 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T23:37:18,802 WARN  [IPC Server handler 4 on default port 46275 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T23:37:18,802 WARN  [IPC Server handler 4 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T23:37:18,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741852_1035 (size=12506)
2024-11-24T23:37:19,207 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/3b9acc9f10e24100a7547bb0375371ad
2024-11-24T23:37:19,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/3b9acc9f10e24100a7547bb0375371ad as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/3b9acc9f10e24100a7547bb0375371ad
2024-11-24T23:37:19,222 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/3b9acc9f10e24100a7547bb0375371ad, entries=7, sequenceid=24, filesize=12.2 K
2024-11-24T23:37:19,224 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 60a2fed0d024369a8ac05ba72076bbc9 in 438ms, sequenceid=24, compaction requested=false
2024-11-24T23:37:19,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 60a2fed0d024369a8ac05ba72076bbc9:

2024-11-24T23:37:19,224 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-24T23:37:19,224 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:37:19,224 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/3b9acc9f10e24100a7547bb0375371ad because midkey is the same as first or last row
2024-11-24T23:37:20,052 WARN  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]
2024-11-24T23:37:20,052 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,053 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C34929%2C1732491418629:(num 1732491436027) roll requested
2024-11-24T23:37:20,053 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C34929%2C1732491418629.1732491440053
2024-11-24T23:37:20,057 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,058 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK], DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:20,058 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741853_1036
2024-11-24T23:37:20,059 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
2024-11-24T23:37:20,061 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,061 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:20,061 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741854_1037
2024-11-24T23:37:20,062 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]
2024-11-24T23:37:20,065 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43243
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51040 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741855_1038] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10]'}, localName='127.0.0.1:37325', datanodeUuid='bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3', xmitsInProgress=0}:Exception transferring block BP-1558194410-172.17.0.3-1732491416060:blk_1073741855_1038 to mirror 127.0.0.1:43243
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:20,065 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]) is bad.
2024-11-24T23:37:20,065 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741855_1038
2024-11-24T23:37:20,065 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51040 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741855_1038] {}] datanode.BlockReceiver(316): Block 1073741855 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T23:37:20,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51040 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741855_1038] {}] datanode.DataXceiver(331): 127.0.0.1:37325:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51040 dst: /127.0.0.1:37325
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:20,066 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]
2024-11-24T23:37:20,068 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35595
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,068 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51046 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10]'}, localName='127.0.0.1:37325', datanodeUuid='bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3', xmitsInProgress=0}:Exception transferring block BP-1558194410-172.17.0.3-1732491416060:blk_1073741856_1039 to mirror 127.0.0.1:35595
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:20,068 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK], DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:20,068 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741856_1039
2024-11-24T23:37:20,068 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51046 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T23:37:20,068 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51046 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:37325:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51046 dst: /127.0.0.1:37325
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:20,069 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:20,070 WARN  [IPC Server handler 2 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T23:37:20,070 WARN  [IPC Server handler 2 on default port 46275 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T23:37:20,070 WARN  [IPC Server handler 2 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T23:37:20,072 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:20,072 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:20,073 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:20,073 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:20,073 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:20,073 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491436027 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491440053
2024-11-24T23:37:20,074 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44357:44357)]
2024-11-24T23:37:20,074 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 is not closed yet, will try archiving it next time
2024-11-24T23:37:20,074 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491436027 is not closed yet, will try archiving it next time
2024-11-24T23:37:20,074 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491432017 to hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/oldWALs/ba7f32532130%2C34929%2C1732491418629.1732491432017
2024-11-24T23:37:20,075 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741842_1025 (size=25992)
2024-11-24T23:37:20,107 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] regionserver.HRegion(8855): Flush requested on 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:20,218 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 60a2fed0d024369a8ac05ba72076bbc9 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-24T23:37:20,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/e5eb6aa764cd4b8a8b044d185002875b is 1079, key is tmprow/info:/1732491440215/Put/seqid=0
2024-11-24T23:37:20,227 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,227 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:20,227 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741858_1041
2024-11-24T23:37:20,228 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:20,229 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,229 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:20,229 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741859_1042
2024-11-24T23:37:20,230 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
2024-11-24T23:37:20,233 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43597
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,233 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51052 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10]'}, localName='127.0.0.1:37325', datanodeUuid='bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3', xmitsInProgress=0}:Exception transferring block BP-1558194410-172.17.0.3-1732491416060:blk_1073741860_1043 to mirror 127.0.0.1:43597
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:20,233 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK], DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:20,233 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741860_1043
2024-11-24T23:37:20,233 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51052 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-24T23:37:20,233 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51052 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:37325:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51052 dst: /127.0.0.1:37325
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:20,234 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]
2024-11-24T23:37:20,237 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43243
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,237 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51056 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741861_1044] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10]'}, localName='127.0.0.1:37325', datanodeUuid='bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3', xmitsInProgress=0}:Exception transferring block BP-1558194410-172.17.0.3-1732491416060:blk_1073741861_1044 to mirror 127.0.0.1:43243
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:20,237 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]) is bad.
2024-11-24T23:37:20,237 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51056 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741861_1044] {}] datanode.BlockReceiver(316): Block 1073741861 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-24T23:37:20,237 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741861_1044
2024-11-24T23:37:20,237 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:51056 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741861_1044] {}] datanode.DataXceiver(331): 127.0.0.1:37325:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51056 dst: /127.0.0.1:37325
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:20,238 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]
2024-11-24T23:37:20,239 WARN  [IPC Server handler 1 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T23:37:20,239 WARN  [IPC Server handler 1 on default port 46275 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T23:37:20,239 WARN  [IPC Server handler 1 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T23:37:20,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741862_1045 (size=6027)
2024-11-24T23:37:20,476 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 is not closed yet, will try archiving it next time
2024-11-24T23:37:20,644 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/e5eb6aa764cd4b8a8b044d185002875b
2024-11-24T23:37:20,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/e5eb6aa764cd4b8a8b044d185002875b as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/e5eb6aa764cd4b8a8b044d185002875b
2024-11-24T23:37:20,658 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/e5eb6aa764cd4b8a8b044d185002875b, entries=1, sequenceid=34, filesize=5.9 K
2024-11-24T23:37:20,659 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 60a2fed0d024369a8ac05ba72076bbc9 in 442ms, sequenceid=34, compaction requested=true
2024-11-24T23:37:20,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 60a2fed0d024369a8ac05ba72076bbc9:

2024-11-24T23:37:20,659 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-24T23:37:20,659 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:37:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/3b9acc9f10e24100a7547bb0375371ad because midkey is the same as first or last row
2024-11-24T23:37:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 60a2fed0d024369a8ac05ba72076bbc9:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:37:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:37:20,660 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:37:20,661 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:37:20,662 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HStore(1541): 60a2fed0d024369a8ac05ba72076bbc9/info is initiating minor compaction (all files)
2024-11-24T23:37:20,662 INFO  [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 60a2fed0d024369a8ac05ba72076bbc9/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:20,662 INFO  [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/c117af440de54a74bda89c1b6a8ac1b2, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/3b9acc9f10e24100a7547bb0375371ad, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/e5eb6aa764cd4b8a8b044d185002875b] into tmpdir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp, totalSize=28.2 K
2024-11-24T23:37:20,662 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] compactions.Compactor(225): Compacting c117af440de54a74bda89c1b6a8ac1b2, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732491434085
2024-11-24T23:37:20,663 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b9acc9f10e24100a7547bb0375371ad, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1732491438145
2024-11-24T23:37:20,663 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5eb6aa764cd4b8a8b044d185002875b, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732491440215
2024-11-24T23:37:20,679 INFO  [RS:0;ba7f32532130:34929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 60a2fed0d024369a8ac05ba72076bbc9#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:37:20,679 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/eb63680c4aea4dd7813bc41ddd2e154d is 1080, key is row0002/info:/1732491434085/Put/seqid=0
2024-11-24T23:37:20,681 WARN  [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,681 WARN  [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:20,681 WARN  [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741863_1046
2024-11-24T23:37:20,682 WARN  [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
2024-11-24T23:37:20,683 WARN  [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,683 WARN  [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:20,683 WARN  [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741864_1047
2024-11-24T23:37:20,683 WARN  [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:20,684 WARN  [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,685 WARN  [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:20,685 WARN  [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741865_1048
2024-11-24T23:37:20,685 WARN  [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]
2024-11-24T23:37:20,686 WARN  [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,686 WARN  [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]) is bad.
2024-11-24T23:37:20,686 WARN  [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741866_1049
2024-11-24T23:37:20,687 WARN  [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]
2024-11-24T23:37:20,688 WARN  [IPC Server handler 4 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T23:37:20,688 WARN  [IPC Server handler 4 on default port 46275 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T23:37:20,688 WARN  [IPC Server handler 4 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T23:37:20,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741867_1050 (size=17994)
2024-11-24T23:37:20,747 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:20,917 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7888f9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741847_1030 to 127.0.0.1:42531 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:20,917 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5f7747a8[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741852_1035 to 127.0.0.1:35595 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:21,104 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/eb63680c4aea4dd7813bc41ddd2e154d as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/eb63680c4aea4dd7813bc41ddd2e154d
2024-11-24T23:37:21,113 INFO  [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 60a2fed0d024369a8ac05ba72076bbc9/info of 60a2fed0d024369a8ac05ba72076bbc9 into eb63680c4aea4dd7813bc41ddd2e154d(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:37:21,113 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 60a2fed0d024369a8ac05ba72076bbc9:

2024-11-24T23:37:21,113 INFO  [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9., storeName=60a2fed0d024369a8ac05ba72076bbc9/info, priority=13, startTime=1732491440660; duration=0sec
2024-11-24T23:37:21,113 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-24T23:37:21,114 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:37:21,114 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/eb63680c4aea4dd7813bc41ddd2e154d because midkey is the same as first or last row
2024-11-24T23:37:21,114 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-24T23:37:21,114 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:37:21,114 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/eb63680c4aea4dd7813bc41ddd2e154d because midkey is the same as first or last row
2024-11-24T23:37:21,114 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-24T23:37:21,114 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:37:21,114 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/eb63680c4aea4dd7813bc41ddd2e154d because midkey is the same as first or last row
2024-11-24T23:37:21,114 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:37:21,114 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 60a2fed0d024369a8ac05ba72076bbc9:info
2024-11-24T23:37:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] regionserver.HRegion(8855): Flush requested on 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:21,650 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 60a2fed0d024369a8ac05ba72076bbc9 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-24T23:37:21,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/42e12d2976cd4608b3d10de43066948d is 1079, key is tmprow/info:/1732491441647/Put/seqid=0
2024-11-24T23:37:21,660 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:21,661 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:21,661 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741868_1051
2024-11-24T23:37:21,661 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]
2024-11-24T23:37:21,663 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:21,663 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:21,663 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741869_1052
2024-11-24T23:37:21,664 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:21,666 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:21,666 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:21,666 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741870_1053
2024-11-24T23:37:21,667 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
2024-11-24T23:37:21,668 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:21,668 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]) is bad.
2024-11-24T23:37:21,668 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741871_1054
2024-11-24T23:37:21,669 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]
2024-11-24T23:37:21,670 WARN  [IPC Server handler 0 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T23:37:21,670 WARN  [IPC Server handler 0 on default port 46275 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T23:37:21,670 WARN  [IPC Server handler 0 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T23:37:21,672 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741872_1055 (size=6027)
2024-11-24T23:37:21,919 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5f7747a8[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741842_1025 to 127.0.0.1:42531 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:21,919 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7888f9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741862_1045 to 127.0.0.1:35595 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:22,074 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/42e12d2976cd4608b3d10de43066948d
2024-11-24T23:37:22,074 WARN  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]
2024-11-24T23:37:22,075 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:22,075 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C34929%2C1732491418629:(num 1732491440053) roll requested
2024-11-24T23:37:22,075 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C34929%2C1732491418629.1732491442075
2024-11-24T23:37:22,079 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:22,080 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:22,080 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741873_1056
2024-11-24T23:37:22,081 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
2024-11-24T23:37:22,084 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43243
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:22,084 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49220 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741874_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10]'}, localName='127.0.0.1:37325', datanodeUuid='bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3', xmitsInProgress=0}:Exception transferring block BP-1558194410-172.17.0.3-1732491416060:blk_1073741874_1057 to mirror 127.0.0.1:43243
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:22,085 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]) is bad.
2024-11-24T23:37:22,085 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741874_1057
2024-11-24T23:37:22,085 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49220 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741874_1057] {}] datanode.BlockReceiver(316): Block 1073741874 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T23:37:22,085 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49220 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741874_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37325:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49220 dst: /127.0.0.1:37325
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:22,085 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]
2024-11-24T23:37:22,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/42e12d2976cd4608b3d10de43066948d as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/42e12d2976cd4608b3d10de43066948d
2024-11-24T23:37:22,087 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:22,087 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:22,087 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741875_1058
2024-11-24T23:37:22,088 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:22,090 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43597
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:22,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49234 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10]'}, localName='127.0.0.1:37325', datanodeUuid='bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3', xmitsInProgress=0}:Exception transferring block BP-1558194410-172.17.0.3-1732491416060:blk_1073741876_1059 to mirror 127.0.0.1:43597
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:22,090 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK], DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:22,090 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49234 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T23:37:22,090 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741876_1059
2024-11-24T23:37:22,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49234 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:37325:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49234 dst: /127.0.0.1:37325
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:22,091 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]
2024-11-24T23:37:22,092 WARN  [IPC Server handler 2 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T23:37:22,092 WARN  [IPC Server handler 2 on default port 46275 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T23:37:22,092 WARN  [IPC Server handler 2 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T23:37:22,093 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/42e12d2976cd4608b3d10de43066948d, entries=1, sequenceid=45, filesize=5.9 K
2024-11-24T23:37:22,094 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 60a2fed0d024369a8ac05ba72076bbc9 in 444ms, sequenceid=45, compaction requested=false
2024-11-24T23:37:22,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 60a2fed0d024369a8ac05ba72076bbc9:

2024-11-24T23:37:22,094 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-24T23:37:22,095 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:37:22,095 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/eb63680c4aea4dd7813bc41ddd2e154d because midkey is the same as first or last row
2024-11-24T23:37:22,095 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:22,095 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:22,095 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:22,095 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:22,095 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:22,095 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491440053 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491442075
2024-11-24T23:37:22,097 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741857_1040 (size=13591)
2024-11-24T23:37:22,100 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44357:44357)]
2024-11-24T23:37:22,100 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 is not closed yet, will try archiving it next time
2024-11-24T23:37:22,100 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491440053 is not closed yet, will try archiving it next time
2024-11-24T23:37:22,100 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491436027 to hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/oldWALs/ba7f32532130%2C34929%2C1732491418629.1732491436027
2024-11-24T23:37:22,108 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:22,499 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 is not closed yet, will try archiving it next time
2024-11-24T23:37:22,747 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] regionserver.HRegion(8855): Flush requested on 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:23,092 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 60a2fed0d024369a8ac05ba72076bbc9 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-24T23:37:23,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/5c999ca475894a6a9361d4625cc7beda is 1079, key is tmprow/info:/1732491443089/Put/seqid=0
2024-11-24T23:37:23,100 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:23,101 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:23,101 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741878_1061
2024-11-24T23:37:23,101 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:23,102 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:23,102 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:23,102 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741879_1062
2024-11-24T23:37:23,103 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
2024-11-24T23:37:23,104 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:23,104 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:23,104 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741880_1063
2024-11-24T23:37:23,104 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]
2024-11-24T23:37:23,106 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43243
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:23,106 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49264 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741881_1064] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10]'}, localName='127.0.0.1:37325', datanodeUuid='bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3', xmitsInProgress=0}:Exception transferring block BP-1558194410-172.17.0.3-1732491416060:blk_1073741881_1064 to mirror 127.0.0.1:43243
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:23,106 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]) is bad.
2024-11-24T23:37:23,106 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741881_1064
2024-11-24T23:37:23,106 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49264 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741881_1064] {}] datanode.BlockReceiver(316): Block 1073741881 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-24T23:37:23,106 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49264 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741881_1064] {}] datanode.DataXceiver(331): 127.0.0.1:37325:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49264 dst: /127.0.0.1:37325
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:23,107 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]
2024-11-24T23:37:23,108 WARN  [IPC Server handler 2 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T23:37:23,108 WARN  [IPC Server handler 2 on default port 46275 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T23:37:23,108 WARN  [IPC Server handler 2 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T23:37:23,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741882_1065 (size=6027)
2024-11-24T23:37:23,512 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/5c999ca475894a6a9361d4625cc7beda
2024-11-24T23:37:23,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/5c999ca475894a6a9361d4625cc7beda as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/5c999ca475894a6a9361d4625cc7beda
2024-11-24T23:37:23,533 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/5c999ca475894a6a9361d4625cc7beda, entries=1, sequenceid=55, filesize=5.9 K
2024-11-24T23:37:23,534 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 60a2fed0d024369a8ac05ba72076bbc9 in 442ms, sequenceid=55, compaction requested=true
2024-11-24T23:37:23,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 60a2fed0d024369a8ac05ba72076bbc9:

2024-11-24T23:37:23,534 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-24T23:37:23,534 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:37:23,534 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/eb63680c4aea4dd7813bc41ddd2e154d because midkey is the same as first or last row
2024-11-24T23:37:23,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 60a2fed0d024369a8ac05ba72076bbc9:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:37:23,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:37:23,534 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:37:23,536 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:37:23,536 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HStore(1541): 60a2fed0d024369a8ac05ba72076bbc9/info is initiating minor compaction (all files)
2024-11-24T23:37:23,536 INFO  [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 60a2fed0d024369a8ac05ba72076bbc9/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:23,536 INFO  [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/eb63680c4aea4dd7813bc41ddd2e154d, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/42e12d2976cd4608b3d10de43066948d, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/5c999ca475894a6a9361d4625cc7beda] into tmpdir=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp, totalSize=29.3 K
2024-11-24T23:37:23,537 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb63680c4aea4dd7813bc41ddd2e154d, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732491434085
2024-11-24T23:37:23,538 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42e12d2976cd4608b3d10de43066948d, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732491441647
2024-11-24T23:37:23,538 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c999ca475894a6a9361d4625cc7beda, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732491443089
2024-11-24T23:37:23,559 INFO  [RS:0;ba7f32532130:34929-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 60a2fed0d024369a8ac05ba72076bbc9#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:37:23,559 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/c987608c6a274647b0af2b66cb86c64c is 1080, key is row0002/info:/1732491434085/Put/seqid=0
2024-11-24T23:37:23,561 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:23,561 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK], DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]) is bad.
2024-11-24T23:37:23,562 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741883_1066
2024-11-24T23:37:23,562 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42531,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]
2024-11-24T23:37:23,563 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:23,563 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:23,563 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741884_1067
2024-11-24T23:37:23,564 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:23,565 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:23,565 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]) is bad.
2024-11-24T23:37:23,565 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741885_1068
2024-11-24T23:37:23,566 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]
2024-11-24T23:37:23,567 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43597
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:23,567 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49292 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741886_1069] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10]'}, localName='127.0.0.1:37325', datanodeUuid='bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3', xmitsInProgress=0}:Exception transferring block BP-1558194410-172.17.0.3-1732491416060:blk_1073741886_1069 to mirror 127.0.0.1:43597
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:23,568 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK], DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:23,568 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741886_1069
2024-11-24T23:37:23,568 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49292 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741886_1069] {}] datanode.BlockReceiver(316): Block 1073741886 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-24T23:37:23,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_977115584_22 at /127.0.0.1:49292 [Receiving block BP-1558194410-172.17.0.3-1732491416060:blk_1073741886_1069] {}] datanode.DataXceiver(331): 127.0.0.1:37325:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49292 dst: /127.0.0.1:37325
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:23,568 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]
2024-11-24T23:37:23,569 WARN  [IPC Server handler 2 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T23:37:23,569 WARN  [IPC Server handler 2 on default port 46275 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T23:37:23,569 WARN  [IPC Server handler 2 on default port 46275 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T23:37:23,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741887_1070 (size=18097)
2024-11-24T23:37:23,919 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7888f9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741867_1050 to 127.0.0.1:35595 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:23,919 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5f7747a8[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741872_1055 to 127.0.0.1:43597 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:23,986 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/c987608c6a274647b0af2b66cb86c64c as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/c987608c6a274647b0af2b66cb86c64c
2024-11-24T23:37:23,995 INFO  [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 60a2fed0d024369a8ac05ba72076bbc9/info of 60a2fed0d024369a8ac05ba72076bbc9 into c987608c6a274647b0af2b66cb86c64c(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 60a2fed0d024369a8ac05ba72076bbc9:

2024-11-24T23:37:23,996 INFO  [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9., storeName=60a2fed0d024369a8ac05ba72076bbc9/info, priority=13, startTime=1732491443534; duration=0sec
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/c987608c6a274647b0af2b66cb86c64c because midkey is the same as first or last row
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/c987608c6a274647b0af2b66cb86c64c because midkey is the same as first or last row
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/c987608c6a274647b0af2b66cb86c64c because midkey is the same as first or last row
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:37:23,996 DEBUG [RS:0;ba7f32532130:34929-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 60a2fed0d024369a8ac05ba72076bbc9:info
2024-11-24T23:37:24,101 WARN  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-24T23:37:24,101 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:24,108 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:24,124 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:37:24,128 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:37:24,129 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:37:24,129 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:37:24,129 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:37:24,129 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d6a3337{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:37:24,130 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a8eeeb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:37:24,226 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@122a196d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/java.io.tmpdir/jetty-localhost-41499-hadoop-hdfs-3_4_1-tests_jar-_-any-3213871301797806743/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:24,226 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a528add{HTTP/1.1, (http/1.1)}{localhost:41499}
2024-11-24T23:37:24,227 INFO  [Time-limited test {}] server.Server(415): Started @138810ms
2024-11-24T23:37:24,228 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:37:24,681 WARN  [Thread-990 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:37:24,689 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12ddc2f50ce6cdf5 with lease ID 0xb89acf9f033a83ef: from storage DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa node DatanodeRegistration(127.0.0.1:46319, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=38079, infoSecurePort=0, ipcPort=41757, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:37:24,689 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12ddc2f50ce6cdf5 with lease ID 0xb89acf9f033a83ef: from storage DS-267b4a32-f8a8-4a4a-90f0-53ff8a5ecd0d node DatanodeRegistration(127.0.0.1:46319, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=38079, infoSecurePort=0, ipcPort=41757, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T23:37:24,748 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:24,920 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5f7747a8[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741857_1040 to 127.0.0.1:43597 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:24,920 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7888f9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741882_1065 to 127.0.0.1:43243 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:26,101 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:26,108 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:26,748 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:26,918 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7888f9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37325, datanodeUuid=bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3, infoPort=44357, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741887_1070 to 127.0.0.1:43597 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:28,102 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:28,109 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:28,441 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T23:37:28,749 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:29,030 ERROR [FSHLog-0-hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData-prefix:ba7f32532130,45207,1732491418462 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:29,030 WARN  [FSHLog-0-hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData-prefix:ba7f32532130,45207,1732491418462 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:29,031 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C45207%2C1732491418462:(num 1732491418767) roll requested
2024-11-24T23:37:29,031 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C45207%2C1732491418462.1732491449031
2024-11-24T23:37:29,034 WARN  [Thread-1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1071
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:29,034 WARN  [Thread-1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741888_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]) is bad.
2024-11-24T23:37:29,034 WARN  [Thread-1010 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741888_1071
2024-11-24T23:37:29,035 WARN  [Thread-1010 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43597,DS-b015f799-15b8-418e-99d4-9f95a934fe13,DISK]
2024-11-24T23:37:29,036 WARN  [Thread-1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1072
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:29,036 WARN  [Thread-1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741889_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:29,036 WARN  [Thread-1010 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741889_1072
2024-11-24T23:37:29,037 WARN  [Thread-1010 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:29,038 WARN  [Thread-1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1073
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:29,038 WARN  [Thread-1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741890_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK], DatanodeInfoWithStorage[127.0.0.1:46319,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]) is bad.
2024-11-24T23:37:29,038 WARN  [Thread-1010 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741890_1073
2024-11-24T23:37:29,038 WARN  [Thread-1010 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]
2024-11-24T23:37:29,042 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:29,043 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:29,043 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:29,043 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:29,043 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:29,043 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462/ba7f32532130%2C45207%2C1732491418462.1732491418767 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462/ba7f32532130%2C45207%2C1732491418462.1732491449031
2024-11-24T23:37:29,043 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:29,044 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:29,044 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462/ba7f32532130%2C45207%2C1732491418462.1732491418767
2024-11-24T23:37:29,044 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38079:38079),(127.0.0.1/127.0.0.1:44357:44357)]
2024-11-24T23:37:29,044 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462/ba7f32532130%2C45207%2C1732491418462.1732491418767 is not closed yet, will try archiving it next time
2024-11-24T23:37:29,045 WARN  [IPC Server handler 0 on default port 46275 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462/ba7f32532130%2C45207%2C1732491418462.1732491418767 has not been closed. Lease recovery is in progress. RecoveryId = 1075 for block blk_1073741830_1006
2024-11-24T23:37:29,045 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462/ba7f32532130%2C45207%2C1732491418462.1732491418767 after 1ms
2024-11-24T23:37:30,102 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:30,109 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:32,103 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:32,109 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:33,047 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/WALs/ba7f32532130,45207,1732491418462/ba7f32532130%2C45207%2C1732491418462.1732491418767 after 4003ms
2024-11-24T23:37:33,685 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@8019efa[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46319, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=38079, infoSecurePort=0, ipcPort=41757, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741836_1012 to 127.0.0.1:35595 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:33,685 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@759272bf[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46319, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=38079, infoSecurePort=0, ipcPort=41757, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741832_1008 to 127.0.0.1:35595 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:34,103 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:34,110 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:34,684 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@759272bf[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46319, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=38079, infoSecurePort=0, ipcPort=41757, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741828_1004 to 127.0.0.1:43597 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:34,684 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@8019efa[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46319, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=38079, infoSecurePort=0, ipcPort=41757, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741826_1002 to 127.0.0.1:35595 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:34,703 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3972a23c {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1558194410-172.17.0.3-1732491416060:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:43243,null,null])
java.net.ConnectException: Call From ba7f32532130/172.17.0.3 to localhost:45891 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-24T23:37:34,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741833_1019 (size=455)
2024-11-24T23:37:35,058 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491419283 to hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/oldWALs/ba7f32532130%2C34929%2C1732491418629.1732491419283
2024-11-24T23:37:35,059 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491440053 to hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/oldWALs/ba7f32532130%2C34929%2C1732491418629.1732491440053
2024-11-24T23:37:36,103 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:36,110 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:36,685 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@759272bf[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46319, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=38079, infoSecurePort=0, ipcPort=41757, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741833_1019 to 127.0.0.1:43597 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:36,691 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:37:37,971 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C34929%2C1732491418629.1732491457970
2024-11-24T23:37:37,982 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:37,988 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:37,988 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:37,988 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:37,988 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:37,989 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491442075 with entries=12, filesize=11.46 KB; new WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.1732491457970
2024-11-24T23:37:37,990 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741877_1060 (size=11743)
2024-11-24T23:37:37,992 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44357:44357),(127.0.0.1/127.0.0.1:38079:38079)]
2024-11-24T23:37:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34929 {}] regionserver.HRegion(8855): Flush requested on 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:38,007 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 60a2fed0d024369a8ac05ba72076bbc9 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-24T23:37:38,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/22617803a06342619f18b74e77497e82 is 1080, key is row0013/info:/1732491457994/Put/seqid=0
2024-11-24T23:37:38,067 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741893_1077 (size=9267)
2024-11-24T23:37:38,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741893_1077 (size=9267)
2024-11-24T23:37:38,076 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/22617803a06342619f18b74e77497e82
2024-11-24T23:37:38,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/22617803a06342619f18b74e77497e82 as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/22617803a06342619f18b74e77497e82
2024-11-24T23:37:38,104 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-24T23:37:38,104 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:38,110 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:38,113 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/22617803a06342619f18b74e77497e82, entries=4, sequenceid=66, filesize=9.0 K
2024-11-24T23:37:38,115 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7528, heapSize ~8.11 KB/8304, currentSize=8.41 KB/8607 for 60a2fed0d024369a8ac05ba72076bbc9 in 108ms, sequenceid=66, compaction requested=false
2024-11-24T23:37:38,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 60a2fed0d024369a8ac05ba72076bbc9:

2024-11-24T23:37:38,116 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=26.7 K, sizeToCheck=16.0 K
2024-11-24T23:37:38,116 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:37:38,116 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/c987608c6a274647b0af2b66cb86c64c because midkey is the same as first or last row
2024-11-24T23:37:38,276 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T23:37:38,276 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:37:38,276 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:37:38,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:37:38,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:37:38,277 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T23:37:38,277 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=821688174, stopped=false
2024-11-24T23:37:38,277 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ba7f32532130,45207,1732491418462
2024-11-24T23:37:38,279 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T23:37:38,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46117-0x1016f3516f70002, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:37:38,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46117-0x1016f3516f70002, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:38,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:37:38,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:38,324 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:37:38,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:37:38,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:38,325 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:37:38,325 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:37:38,325 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46117-0x1016f3516f70002, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:37:38,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:37:38,325 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:37:38,326 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba7f32532130,34929,1732491418629' *****
2024-11-24T23:37:38,326 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T23:37:38,326 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba7f32532130,46117,1732491419885' *****
2024-11-24T23:37:38,326 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T23:37:38,326 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:37:38,326 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T23:37:38,327 INFO  [RS:1;ba7f32532130:46117 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T23:37:38,327 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T23:37:38,327 INFO  [RS:1;ba7f32532130:46117 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T23:37:38,327 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer(959): stopping server ba7f32532130,46117,1732491419885
2024-11-24T23:37:38,327 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T23:37:38,327 INFO  [RS:1;ba7f32532130:46117 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:37:38,327 INFO  [RS:0;ba7f32532130:34929 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T23:37:38,327 INFO  [RS:1;ba7f32532130:46117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ba7f32532130:46117.
2024-11-24T23:37:38,327 INFO  [RS:0;ba7f32532130:34929 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T23:37:38,327 DEBUG [RS:1;ba7f32532130:46117 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:37:38,327 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(3091): Received CLOSE for 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:38,327 DEBUG [RS:1;ba7f32532130:46117 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:37:38,327 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer(976): stopping server ba7f32532130,46117,1732491419885; all regions closed.
2024-11-24T23:37:38,328 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T23:37:38,332 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(959): stopping server ba7f32532130,34929,1732491418629
2024-11-24T23:37:38,332 INFO  [RS:0;ba7f32532130:34929 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:37:38,333 INFO  [RS:0;ba7f32532130:34929 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ba7f32532130:34929.
2024-11-24T23:37:38,333 DEBUG [RS:0;ba7f32532130:34929 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:37:38,333 DEBUG [RS:0;ba7f32532130:34929 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:37:38,333 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T23:37:38,333 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T23:37:38,333 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T23:37:38,333 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T23:37:38,336 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-24T23:37:38,336 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 60a2fed0d024369a8ac05ba72076bbc9=TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.}
2024-11-24T23:37:38,336 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:38,336 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 60a2fed0d024369a8ac05ba72076bbc9, disabling compactions & flushes
2024-11-24T23:37:38,337 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:38,337 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:38,337 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9. after waiting 0 ms
2024-11-24T23:37:38,337 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:38,337 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 60a2fed0d024369a8ac05ba72076bbc9 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-11-24T23:37:38,339 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:37:38,339 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:37:38,339 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:37:38,340 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:37:38,340 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:37:38,340 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-24T23:37:38,340 ERROR [FSHLog-0-hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7-prefix:ba7f32532130,34929,1732491418629.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:38,341 WARN  [FSHLog-0-hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7-prefix:ba7f32532130,34929,1732491418629.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:38,344 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C34929%2C1732491418629.meta:.meta(num 1732491419738) roll requested
2024-11-24T23:37:38,345 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C34929%2C1732491418629.meta.1732491458344.meta
2024-11-24T23:37:38,349 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:38,352 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:38,356 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:38,359 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:38,359 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:38,361 WARN  [Thread-1036 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:38,361 WARN  [Thread-1036 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:38,361 WARN  [Thread-1036 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741894_1078
2024-11-24T23:37:38,368 WARN  [Thread-1036 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:38,368 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:38,368 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:38,368 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
2024-11-24T23:37:38,376 WARN  [IPC Server handler 3 on default port 46275 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238 has not been closed. Lease recovery is in progress. RecoveryId = 1080 for block blk_1073741837_1013
2024-11-24T23:37:38,384 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238 after 16ms
2024-11-24T23:37:38,391 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/a5690daa6917485f9664cb4b2e06e174 is 1080, key is row0016/info:/1732491458013/Put/seqid=0
2024-11-24T23:37:38,397 WARN  [Thread-1038 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1081
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:38,398 WARN  [Thread-1038 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741896_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:37325,DS-bd38573c-39c9-41fc-9860-aac18083f853,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:38,398 WARN  [Thread-1038 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741896_1081
2024-11-24T23:37:38,401 WARN  [Thread-1038 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:38,420 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:38,424 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:38,428 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:38,431 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:38,432 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:38,432 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491458344.meta
2024-11-24T23:37:38,445 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:38,446 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43243,DS-9812ca79-818f-453d-be00-5ed5f75553cb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:38,446 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
2024-11-24T23:37:38,447 WARN  [IPC Server handler 2 on default port 46275 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta has not been closed. Lease recovery is in progress. RecoveryId = 1083 for block blk_1073741834_1010
2024-11-24T23:37:38,447 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta after 1ms
2024-11-24T23:37:38,448 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741897_1082 (size=13583)
2024-11-24T23:37:38,448 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741897_1082 (size=13583)
2024-11-24T23:37:38,456 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38079:38079),(127.0.0.1/127.0.0.1:44357:44357)]
2024-11-24T23:37:38,456 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta is not closed yet, will try archiving it next time
2024-11-24T23:37:38,525 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/.tmp/info/bf2c0b1482684470b086291dbfde0c62 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9./info:regioninfo/1732491420510/Put/seqid=0
2024-11-24T23:37:38,538 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:38,560 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741898_1084 (size=7089)
2024-11-24T23:37:38,561 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/.tmp/info/bf2c0b1482684470b086291dbfde0c62
2024-11-24T23:37:38,564 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741898_1084 (size=7089)
2024-11-24T23:37:38,626 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/.tmp/ns/43640419b73e45188839276e69c6c2fd is 43, key is default/ns:d/1732491419832/Put/seqid=0
2024-11-24T23:37:38,665 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741899_1085 (size=5153)
2024-11-24T23:37:38,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741899_1085 (size=5153)
2024-11-24T23:37:38,739 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 60a2fed0d024369a8ac05ba72076bbc9
2024-11-24T23:37:38,849 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/a5690daa6917485f9664cb4b2e06e174
2024-11-24T23:37:38,858 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/.tmp/info/a5690daa6917485f9664cb4b2e06e174 as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/a5690daa6917485f9664cb4b2e06e174
2024-11-24T23:37:38,867 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/a5690daa6917485f9664cb4b2e06e174, entries=8, sequenceid=77, filesize=13.3 K
2024-11-24T23:37:38,868 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8607, heapSize ~9.23 KB/9456, currentSize=0 B/0 for 60a2fed0d024369a8ac05ba72076bbc9 in 531ms, sequenceid=77, compaction requested=true
2024-11-24T23:37:38,869 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/c117af440de54a74bda89c1b6a8ac1b2, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/3b9acc9f10e24100a7547bb0375371ad, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/eb63680c4aea4dd7813bc41ddd2e154d, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/e5eb6aa764cd4b8a8b044d185002875b, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/42e12d2976cd4608b3d10de43066948d, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/5c999ca475894a6a9361d4625cc7beda] to archive
2024-11-24T23:37:38,870 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T23:37:38,876 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/c117af440de54a74bda89c1b6a8ac1b2 to hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/c117af440de54a74bda89c1b6a8ac1b2
2024-11-24T23:37:38,883 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/3b9acc9f10e24100a7547bb0375371ad to hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/3b9acc9f10e24100a7547bb0375371ad
2024-11-24T23:37:38,886 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/eb63680c4aea4dd7813bc41ddd2e154d to hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/eb63680c4aea4dd7813bc41ddd2e154d
2024-11-24T23:37:38,890 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/e5eb6aa764cd4b8a8b044d185002875b to hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/e5eb6aa764cd4b8a8b044d185002875b
2024-11-24T23:37:38,892 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/42e12d2976cd4608b3d10de43066948d to hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/42e12d2976cd4608b3d10de43066948d
2024-11-24T23:37:38,895 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/5c999ca475894a6a9361d4625cc7beda to hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/info/5c999ca475894a6a9361d4625cc7beda
2024-11-24T23:37:38,895 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ba7f32532130:45207 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-24T23:37:38,896 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [c117af440de54a74bda89c1b6a8ac1b2=10347, 3b9acc9f10e24100a7547bb0375371ad=12506, eb63680c4aea4dd7813bc41ddd2e154d=17994, e5eb6aa764cd4b8a8b044d185002875b=6027, 42e12d2976cd4608b3d10de43066948d=6027, 5c999ca475894a6a9361d4625cc7beda=6027]
2024-11-24T23:37:38,912 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/60a2fed0d024369a8ac05ba72076bbc9/recovered.edits/80.seqid, newMaxSeqId=80, maxSeqId=1
2024-11-24T23:37:38,914 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:38,914 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 60a2fed0d024369a8ac05ba72076bbc9:
Waiting for close lock at 1732491458336Running coprocessor pre-close hooks at 1732491458336Disabling compacts and flushes for region at 1732491458336Disabling writes for close at 1732491458337 (+1 ms)Obtaining lock to block concurrent updates at 1732491458337Preparing flush snapshotting stores in 60a2fed0d024369a8ac05ba72076bbc9 at 1732491458337Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9., syncing WAL and waiting on mvcc, flushsize=dataSize=8607, getHeapSize=9456, getOffHeapSize=0, getCellsCount=8 at 1732491458338 (+1 ms)Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9. at 1732491458356 (+18 ms)Flushing 60a2fed0d024369a8ac05ba72076bbc9/info: creating writer at 1732491458357 (+1 ms)Flushing 60a2fed0d024369a8ac05ba72076bbc9/info: appending metadata at 1732491458389 (+32 ms)Flushing 60a2fed0d024369a8ac05ba72076bbc9/info: closing flushed file at 1732491458389Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@699848b0: reopening flushed file at 1732491458857 (+468 ms)Finished flush of dataSize ~8.41 KB/8607, heapSize ~9.23 KB/9456, currentSize=0 B/0 for 60a2fed0d024369a8ac05ba72076bbc9 in 531ms, sequenceid=77, compaction requested=true at 1732491458868 (+11 ms)Writing region close event to WAL at 1732491458904 (+36 ms)Running coprocessor post-close hooks at 1732491458914 (+10 ms)Closed at 1732491458914
2024-11-24T23:37:38,914 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732491420137.60a2fed0d024369a8ac05ba72076bbc9.
2024-11-24T23:37:38,924 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741877_1060 (size=11743)
2024-11-24T23:37:38,939 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-24T23:37:39,066 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/.tmp/ns/43640419b73e45188839276e69c6c2fd
2024-11-24T23:37:39,101 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/.tmp/table/eebfc4fbcd7944b29c0ef3618c711c40 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1732491420522/Put/seqid=0
2024-11-24T23:37:39,106 WARN  [Thread-1063 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1086
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:37:39,106 WARN  [Thread-1063 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1558194410-172.17.0.3-1732491416060:blk_1073741900_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK], DatanodeInfoWithStorage[127.0.0.1:46319,DS-498aa1c7-aaef-45f3-83f9-cd633273ddaa,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]) is bad.
2024-11-24T23:37:39,106 WARN  [Thread-1063 {}] hdfs.DataStreamer(1850): Abandoning BP-1558194410-172.17.0.3-1732491416060:blk_1073741900_1086
2024-11-24T23:37:39,112 WARN  [Thread-1063 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35595,DS-f3f37c3c-a65f-4fbf-9279-0063a5c51969,DISK]
2024-11-24T23:37:39,140 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-24T23:37:39,150 INFO  [regionserver/ba7f32532130:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:37:39,155 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741901_1087 (size=5424)
2024-11-24T23:37:39,157 INFO  [regionserver/ba7f32532130:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-24T23:37:39,157 INFO  [regionserver/ba7f32532130:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-24T23:37:39,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741901_1087 (size=5424)
2024-11-24T23:37:39,164 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/.tmp/table/eebfc4fbcd7944b29c0ef3618c711c40
2024-11-24T23:37:39,172 INFO  [regionserver/ba7f32532130:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-24T23:37:39,172 INFO  [regionserver/ba7f32532130:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-24T23:37:39,264 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/.tmp/info/bf2c0b1482684470b086291dbfde0c62 as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/info/bf2c0b1482684470b086291dbfde0c62
2024-11-24T23:37:39,335 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/info/bf2c0b1482684470b086291dbfde0c62, entries=10, sequenceid=11, filesize=6.9 K
2024-11-24T23:37:39,340 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-24T23:37:39,341 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-24T23:37:39,341 DEBUG [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-24T23:37:39,356 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/.tmp/ns/43640419b73e45188839276e69c6c2fd as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/ns/43640419b73e45188839276e69c6c2fd
2024-11-24T23:37:39,406 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/ns/43640419b73e45188839276e69c6c2fd, entries=2, sequenceid=11, filesize=5.0 K
2024-11-24T23:37:39,411 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/.tmp/table/eebfc4fbcd7944b29c0ef3618c711c40 as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/table/eebfc4fbcd7944b29c0ef3618c711c40
2024-11-24T23:37:39,442 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/table/eebfc4fbcd7944b29c0ef3618c711c40, entries=2, sequenceid=11, filesize=5.3 K
2024-11-24T23:37:39,446 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 1105ms, sequenceid=11, compaction requested=false
2024-11-24T23:37:39,465 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-24T23:37:39,466 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:37:39,466 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:37:39,466 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491458339Running coprocessor pre-close hooks at 1732491458339Disabling compacts and flushes for region at 1732491458339Disabling writes for close at 1732491458340 (+1 ms)Obtaining lock to block concurrent updates at 1732491458340Preparing flush snapshotting stores in 1588230740 at 1732491458340Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1732491458341 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732491458463 (+122 ms)Flushing 1588230740/info: creating writer at 1732491458464 (+1 ms)Flushing 1588230740/info: appending metadata at 1732491458524 (+60 ms)Flushing 1588230740/info: closing flushed file at 1732491458524Flushing 1588230740/ns: creating writer at 1732491458585 (+61 ms)Flushing 1588230740/ns: appending metadata at 1732491458625 (+40 ms)Flushing 1588230740/ns: closing flushed file at 1732491458625Flushing 1588230740/table: creating writer at 1732491459075 (+450 ms)Flushing 1588230740/table: appending metadata at 1732491459101 (+26 ms)Flushing 1588230740/table: closing flushed file at 1732491459101Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c560bf1: reopening flushed file at 1732491459236 (+135 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f9a68ca: reopening flushed file at 1732491459335 (+99 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ce1447b: reopening flushed file at 1732491459407 (+72 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 1105ms, sequenceid=11, compaction requested=false at 1732491459446 (+39 ms)Writing region close event to WAL at 1732491459456 (+10 ms)Running coprocessor post-close hooks at 1732491459466 (+10 ms)Closed at 1732491459466
2024-11-24T23:37:39,467 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T23:37:39,541 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(976): stopping server ba7f32532130,34929,1732491418629; all regions closed.
2024-11-24T23:37:39,546 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:39,547 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:39,547 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:39,547 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:39,547 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:39,556 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741895_1079 (size=825)
2024-11-24T23:37:39,556 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741895_1079 (size=825)
2024-11-24T23:37:39,685 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@759272bf[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46319, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=38079, infoSecurePort=0, ipcPort=41757, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741831_1007 to 127.0.0.1:35595 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:39,685 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@8019efa[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46319, datanodeUuid=14ab36d8-5a0d-4818-be52-baede7a0917f, infoPort=38079, infoSecurePort=0, ipcPort=41757, storageInfo=lv=-57;cid=testClusterID;nsid=131593289;c=1732491416060):Failed to transfer BP-1558194410-172.17.0.3-1732491416060:blk_1073741835_1011 to 127.0.0.1:35595 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:40,108 INFO  [regionserver/ba7f32532130:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:37:40,554 INFO  [master/ba7f32532130:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-24T23:37:40,554 INFO  [master/ba7f32532130:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-24T23:37:40,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:37:40,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:37:42,385 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238 after 4017ms
2024-11-24T23:37:42,448 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta after 4002ms
2024-11-24T23:37:43,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-24T23:37:43,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:37:43,145 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T23:37:43,365 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-24T23:37:43,367 DEBUG [RS:1;ba7f32532130:46117 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/oldWALs
2024-11-24T23:37:43,367 INFO  [RS:1;ba7f32532130:46117 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C46117%2C1732491419885:(num 1732491420238)
2024-11-24T23:37:43,367 DEBUG [RS:1;ba7f32532130:46117 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:37:43,367 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:37:43,368 INFO  [RS:1;ba7f32532130:46117 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:37:43,368 INFO  [RS:1;ba7f32532130:46117 {}] hbase.ChoreService(370): Chore service for: regionserver/ba7f32532130:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T23:37:43,368 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T23:37:43,368 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:37:43,368 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T23:37:43,368 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T23:37:43,368 INFO  [RS:1;ba7f32532130:46117 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:37:43,368 INFO  [RS:1;ba7f32532130:46117 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46117
2024-11-24T23:37:43,390 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:43,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46117-0x1016f3516f70002, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba7f32532130,46117,1732491419885
2024-11-24T23:37:43,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:37:43,428 INFO  [RS:1;ba7f32532130:46117 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:37:43,558 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba7f32532130,46117,1732491419885]
2024-11-24T23:37:43,648 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba7f32532130,46117,1732491419885 already deleted, retry=false
2024-11-24T23:37:43,648 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba7f32532130,46117,1732491419885 expired; onlineServers=1
2024-11-24T23:37:43,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46117-0x1016f3516f70002, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:37:43,658 INFO  [RS:1;ba7f32532130:46117 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:37:43,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46117-0x1016f3516f70002, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:37:43,658 INFO  [RS:1;ba7f32532130:46117 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba7f32532130,46117,1732491419885; zookeeper connection closed.
2024-11-24T23:37:43,658 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@69b71030 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@69b71030
2024-11-24T23:37:43,915 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:43,929 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:43,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:43,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:43,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:43,931 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:43,948 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:43,948 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:44,391 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:44,450 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T23:37:44,463 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:44,464 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:44,464 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:44,464 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:44,464 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:44,465 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:44,469 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:44,472 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:44,548 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-24T23:37:44,552 DEBUG [RS:0;ba7f32532130:34929 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/oldWALs
2024-11-24T23:37:44,552 INFO  [RS:0;ba7f32532130:34929 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C34929%2C1732491418629.meta:.meta(num 1732491458344)
2024-11-24T23:37:44,552 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:44,552 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:44,553 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:44,553 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:44,553 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:44,555 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741892_1076 (size=14682)
2024-11-24T23:37:44,555 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741892_1076 (size=14682)
2024-11-24T23:37:44,558 DEBUG [RS:0;ba7f32532130:34929 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/oldWALs
2024-11-24T23:37:44,558 INFO  [RS:0;ba7f32532130:34929 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C34929%2C1732491418629:(num 1732491457970)
2024-11-24T23:37:44,558 DEBUG [RS:0;ba7f32532130:34929 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:37:44,558 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:37:44,558 INFO  [RS:0;ba7f32532130:34929 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:37:44,558 INFO  [RS:0;ba7f32532130:34929 {}] hbase.ChoreService(370): Chore service for: regionserver/ba7f32532130:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T23:37:44,559 INFO  [RS:0;ba7f32532130:34929 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:37:44,559 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:37:44,559 INFO  [RS:0;ba7f32532130:34929 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34929
2024-11-24T23:37:44,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba7f32532130,34929,1732491418629
2024-11-24T23:37:44,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:37:44,638 INFO  [RS:0;ba7f32532130:34929 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:37:44,711 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@d0b10cc {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1558194410-172.17.0.3-1732491416060:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:43243,null,null])
java.net.ConnectException: Call From ba7f32532130/172.17.0.3 to localhost:45891 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-24T23:37:44,719 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba7f32532130,34929,1732491418629]
2024-11-24T23:37:44,733 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba7f32532130,34929,1732491418629 already deleted, retry=false
2024-11-24T23:37:44,733 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba7f32532130,34929,1732491418629 expired; onlineServers=0
2024-11-24T23:37:44,733 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ba7f32532130,45207,1732491418462' *****
2024-11-24T23:37:44,733 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T23:37:44,734 INFO  [M:0;ba7f32532130:45207 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:37:44,734 INFO  [M:0;ba7f32532130:45207 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:37:44,734 DEBUG [M:0;ba7f32532130:45207 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T23:37:44,734 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T23:37:44,734 DEBUG [M:0;ba7f32532130:45207 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T23:37:44,734 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491419031 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491419031,5,FailOnTimeoutGroup]
2024-11-24T23:37:44,734 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491419031 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491419031,5,FailOnTimeoutGroup]
2024-11-24T23:37:44,734 INFO  [M:0;ba7f32532130:45207 {}] hbase.ChoreService(370): Chore service for: master/ba7f32532130:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T23:37:44,734 INFO  [M:0;ba7f32532130:45207 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:37:44,734 DEBUG [M:0;ba7f32532130:45207 {}] master.HMaster(1795): Stopping service threads
2024-11-24T23:37:44,734 INFO  [M:0;ba7f32532130:45207 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T23:37:44,734 INFO  [M:0;ba7f32532130:45207 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:37:44,734 INFO  [M:0;ba7f32532130:45207 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T23:37:44,735 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T23:37:44,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T23:37:44,744 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-11-24T23:37:44,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:44,744 DEBUG [RegionServerTracker-0 {}] master.ActiveMasterManager(353): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-11-24T23:37:44,745 INFO  [M:0;ba7f32532130:45207 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/.lastflushedseqids
2024-11-24T23:37:44,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741902_1088 (size=130)
2024-11-24T23:37:44,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741902_1088 (size=130)
2024-11-24T23:37:44,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:37:44,820 INFO  [RS:0;ba7f32532130:34929 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:37:44,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34929-0x1016f3516f70001, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:37:44,820 INFO  [RS:0;ba7f32532130:34929 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba7f32532130,34929,1732491418629; zookeeper connection closed.
2024-11-24T23:37:44,820 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@186acd3d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@186acd3d
2024-11-24T23:37:44,820 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-24T23:37:45,165 INFO  [M:0;ba7f32532130:45207 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T23:37:45,166 INFO  [M:0;ba7f32532130:45207 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T23:37:45,166 DEBUG [M:0;ba7f32532130:45207 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:37:45,166 INFO  [M:0;ba7f32532130:45207 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:37:45,166 DEBUG [M:0;ba7f32532130:45207 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:37:45,166 DEBUG [M:0;ba7f32532130:45207 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:37:45,166 DEBUG [M:0;ba7f32532130:45207 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:37:45,166 INFO  [M:0;ba7f32532130:45207 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-11-24T23:37:45,189 DEBUG [M:0;ba7f32532130:45207 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55b38cc483c849e884990bb4974662b5 is 82, key is hbase:meta,,1/info:regioninfo/1732491419775/Put/seqid=0
2024-11-24T23:37:45,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741903_1089 (size=5672)
2024-11-24T23:37:45,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741903_1089 (size=5672)
2024-11-24T23:37:45,198 INFO  [M:0;ba7f32532130:45207 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55b38cc483c849e884990bb4974662b5
2024-11-24T23:37:45,220 DEBUG [M:0;ba7f32532130:45207 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9d031a1ac106483498703c3ca4b38f58 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732491420527/Put/seqid=0
2024-11-24T23:37:45,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741904_1090 (size=6255)
2024-11-24T23:37:45,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741904_1090 (size=6255)
2024-11-24T23:37:45,237 INFO  [M:0;ba7f32532130:45207 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9d031a1ac106483498703c3ca4b38f58
2024-11-24T23:37:45,245 INFO  [M:0;ba7f32532130:45207 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9d031a1ac106483498703c3ca4b38f58
2024-11-24T23:37:45,272 DEBUG [M:0;ba7f32532130:45207 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/94be33a26cd54443bd91500848a3257f is 69, key is ba7f32532130,34929,1732491418629/rs:state/1732491419113/Put/seqid=0
2024-11-24T23:37:45,282 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741905_1091 (size=5224)
2024-11-24T23:37:45,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741905_1091 (size=5224)
2024-11-24T23:37:45,285 INFO  [M:0;ba7f32532130:45207 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/94be33a26cd54443bd91500848a3257f
2024-11-24T23:37:45,314 DEBUG [M:0;ba7f32532130:45207 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fb88615dda4a48e4a75bc83c507ada79 is 52, key is load_balancer_on/state:d/1732491419862/Put/seqid=0
2024-11-24T23:37:45,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741906_1092 (size=5056)
2024-11-24T23:37:45,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741906_1092 (size=5056)
2024-11-24T23:37:45,325 INFO  [M:0;ba7f32532130:45207 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fb88615dda4a48e4a75bc83c507ada79
2024-11-24T23:37:45,334 DEBUG [M:0;ba7f32532130:45207 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/55b38cc483c849e884990bb4974662b5 as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/55b38cc483c849e884990bb4974662b5
2024-11-24T23:37:45,347 INFO  [M:0;ba7f32532130:45207 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/55b38cc483c849e884990bb4974662b5, entries=8, sequenceid=60, filesize=5.5 K
2024-11-24T23:37:45,350 DEBUG [M:0;ba7f32532130:45207 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9d031a1ac106483498703c3ca4b38f58 as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9d031a1ac106483498703c3ca4b38f58
2024-11-24T23:37:45,365 INFO  [M:0;ba7f32532130:45207 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9d031a1ac106483498703c3ca4b38f58
2024-11-24T23:37:45,366 INFO  [M:0;ba7f32532130:45207 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9d031a1ac106483498703c3ca4b38f58, entries=6, sequenceid=60, filesize=6.1 K
2024-11-24T23:37:45,367 DEBUG [M:0;ba7f32532130:45207 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/94be33a26cd54443bd91500848a3257f as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/94be33a26cd54443bd91500848a3257f
2024-11-24T23:37:45,375 INFO  [M:0;ba7f32532130:45207 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/94be33a26cd54443bd91500848a3257f, entries=2, sequenceid=60, filesize=5.1 K
2024-11-24T23:37:45,376 DEBUG [M:0;ba7f32532130:45207 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fb88615dda4a48e4a75bc83c507ada79 as hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fb88615dda4a48e4a75bc83c507ada79
2024-11-24T23:37:45,384 INFO  [M:0;ba7f32532130:45207 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fb88615dda4a48e4a75bc83c507ada79, entries=1, sequenceid=60, filesize=4.9 K
2024-11-24T23:37:45,385 INFO  [M:0;ba7f32532130:45207 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 219ms, sequenceid=60, compaction requested=false
2024-11-24T23:37:45,387 INFO  [M:0;ba7f32532130:45207 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:37:45,387 DEBUG [M:0;ba7f32532130:45207 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491465166Disabling compacts and flushes for region at 1732491465166Disabling writes for close at 1732491465166Obtaining lock to block concurrent updates at 1732491465166Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732491465166Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1732491465167 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732491465168 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732491465168Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732491465189 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732491465189Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732491465206 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732491465220 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732491465220Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732491465245 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732491465271 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732491465271Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732491465293 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732491465313 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732491465313Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@181651b7: reopening flushed file at 1732491465332 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18979c2: reopening flushed file at 1732491465348 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@174b9ae5: reopening flushed file at 1732491465366 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ddb9b8e: reopening flushed file at 1732491465375 (+9 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 219ms, sequenceid=60, compaction requested=false at 1732491465385 (+10 ms)Writing region close event to WAL at 1732491465387 (+2 ms)Closed at 1732491465387
2024-11-24T23:37:45,388 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:45,392 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:45,392 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:45,393 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:45,393 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:45,393 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:37:45,395 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37325 is added to blk_1073741891_1074 (size=1045)
2024-11-24T23:37:45,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46319 is added to blk_1073741891_1074 (size=1045)
2024-11-24T23:37:45,397 INFO  [M:0;ba7f32532130:45207 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T23:37:45,397 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:37:45,397 INFO  [M:0;ba7f32532130:45207 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45207
2024-11-24T23:37:45,398 INFO  [M:0;ba7f32532130:45207 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:37:45,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:45,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:37:45,549 INFO  [M:0;ba7f32532130:45207 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:37:45,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45207-0x1016f3516f70000, quorum=127.0.0.1:52148, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:37:45,552 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@122a196d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:45,552 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a528add{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:37:45,552 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:37:45,553 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a8eeeb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:37:45,553 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d6a3337{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,STOPPED}
2024-11-24T23:37:45,554 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:37:45,554 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:37:45,554 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:37:45,555 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1558194410-172.17.0.3-1732491416060 (Datanode Uuid 14ab36d8-5a0d-4818-be52-baede7a0917f) service to localhost/127.0.0.1:46275
2024-11-24T23:37:45,554 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@595d5d73 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:43243,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:45891 , LocalHost:localPort ba7f32532130/172.17.0.3:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-24T23:37:45,555 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@595d5d73 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:46319,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1558194410-172.17.0.3-1732491416060
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:45,555 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@595d5d73 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:43243,null,null], DatanodeInfoWithStorage[127.0.0.1:46319,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1558194410-172.17.0.3-1732491416060:blk_1073741837_1013, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:43243,null,null], DatanodeInfoWithStorage[127.0.0.1:46319,null,null]]
2024-11-24T23:37:45,555 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@595d5d73 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:43243,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1558194410-172.17.0.3-1732491416060
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:45,555 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@595d5d73 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:46319,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1558194410-172.17.0.3-1732491416060
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:37:45,555 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@595d5d73 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:43243,null,null], DatanodeInfoWithStorage[127.0.0.1:46319,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1558194410-172.17.0.3-1732491416060:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:43243,null,null], DatanodeInfoWithStorage[127.0.0.1:46319,null,null]]
2024-11-24T23:37:45,555 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data3/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:45,556 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data4/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:45,556 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:37:45,559 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@20faceaa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:45,559 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e1ad43e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:37:45,559 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:37:45,559 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47432b7b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:37:45,560 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b21f544{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,STOPPED}
2024-11-24T23:37:45,561 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:37:45,561 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:37:45,561 WARN  [BP-1558194410-172.17.0.3-1732491416060 heartbeating to localhost/127.0.0.1:46275 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1558194410-172.17.0.3-1732491416060 (Datanode Uuid bf9f15b0-4eaa-46d1-b096-f6105bf3ecf3) service to localhost/127.0.0.1:46275
2024-11-24T23:37:45,561 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:37:45,562 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data9/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:45,562 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/cluster_e91c0302-37ef-8990-1198-ad34267cd69e/data/data10/current/BP-1558194410-172.17.0.3-1732491416060 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:37:45,562 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:37:45,568 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7982676d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:37:45,569 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2efbdc75{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:37:45,569 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:37:45,569 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d790455{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:37:45,569 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3150e6db{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir/,STOPPED}
2024-11-24T23:37:45,580 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T23:37:45,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T23:37:45,631 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=158 (was 82)
Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46275
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46275
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:46275
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46275 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46275
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46275 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46275 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:38951
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f0134bf5e20.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46275
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46275
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46275 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38951
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f0134bf5e20.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:46275
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
 - Thread LEAK? -, OpenFileDescriptor=448 (was 405) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=268 (was 233) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4601 (was 2787) - AvailableMemoryMB LEAK? -
2024-11-24T23:37:45,640 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=158, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=4600
2024-11-24T23:37:45,640 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T23:37:45,641 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.log.dir so I do NOT create it in target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261
2024-11-24T23:37:45,641 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c87bb4b8-50f4-39df-13fb-d51c1cb1cf02/hadoop.tmp.dir so I do NOT create it in target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261
2024-11-24T23:37:45,641 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d, deleteOnExit=true
2024-11-24T23:37:45,641 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T23:37:45,641 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/test.cache.data in system properties and HBase conf
2024-11-24T23:37:45,641 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T23:37:45,641 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir in system properties and HBase conf
2024-11-24T23:37:45,641 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T23:37:45,641 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T23:37:45,642 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T23:37:45,642 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T23:37:45,642 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:37:45,642 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:37:45,642 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T23:37:45,642 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:37:45,642 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T23:37:45,642 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T23:37:45,643 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:37:45,643 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:37:45,643 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T23:37:45,643 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/nfs.dump.dir in system properties and HBase conf
2024-11-24T23:37:45,643 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/java.io.tmpdir in system properties and HBase conf
2024-11-24T23:37:45,643 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:37:45,643 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T23:37:45,643 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T23:37:45,660 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:37:45,996 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:37:46,001 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:37:46,004 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:37:46,005 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:37:46,005 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:37:46,006 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:37:46,006 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7e13d66c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:37:46,007 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e4cbcc2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:37:46,105 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f78fbde{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/java.io.tmpdir/jetty-localhost-34339-hadoop-hdfs-3_4_1-tests_jar-_-any-14353116464303905983/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:37:46,105 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53effb5a{HTTP/1.1, (http/1.1)}{localhost:34339}
2024-11-24T23:37:46,105 INFO  [Time-limited test {}] server.Server(415): Started @160689ms
2024-11-24T23:37:46,123 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:37:46,393 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:46,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:46,488 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:37:46,493 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:37:46,499 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:37:46,499 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:37:46,499 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:37:46,500 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a7cb65f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:37:46,500 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3bfb2c17{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:37:46,645 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@51031d29{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/java.io.tmpdir/jetty-localhost-37397-hadoop-hdfs-3_4_1-tests_jar-_-any-1843102890997844212/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:46,645 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1a48749e{HTTP/1.1, (http/1.1)}{localhost:37397}
2024-11-24T23:37:46,646 INFO  [Time-limited test {}] server.Server(415): Started @161230ms
2024-11-24T23:37:46,648 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:37:46,693 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:37:46,699 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:37:46,700 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:37:46,701 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:37:46,701 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:37:46,701 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41d5af00{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:37:46,702 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@59ddfe6e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:37:46,803 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@716d71e9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/java.io.tmpdir/jetty-localhost-44979-hadoop-hdfs-3_4_1-tests_jar-_-any-12912793864211904714/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:37:46,803 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@60deb4a2{HTTP/1.1, (http/1.1)}{localhost:44979}
2024-11-24T23:37:46,803 INFO  [Time-limited test {}] server.Server(415): Started @161387ms
2024-11-24T23:37:46,805 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:37:47,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:47,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:47,944 WARN  [Thread-1195 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data2/current/BP-1556390239-172.17.0.3-1732491465666/current, will proceed with Du for space computation calculation, 
2024-11-24T23:37:47,944 WARN  [Thread-1194 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data1/current/BP-1556390239-172.17.0.3-1732491465666/current, will proceed with Du for space computation calculation, 
2024-11-24T23:37:47,970 WARN  [Thread-1158 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:37:47,973 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf55933fe6b0fe958 with lease ID 0x3ad51377744934fb: Processing first storage report for DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13 from datanode DatanodeRegistration(127.0.0.1:46187, datanodeUuid=503b6869-be40-49d2-984d-e9a7c62761c2, infoPort=33999, infoSecurePort=0, ipcPort=42219, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666)
2024-11-24T23:37:47,973 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf55933fe6b0fe958 with lease ID 0x3ad51377744934fb: from storage DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13 node DatanodeRegistration(127.0.0.1:46187, datanodeUuid=503b6869-be40-49d2-984d-e9a7c62761c2, infoPort=33999, infoSecurePort=0, ipcPort=42219, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:37:47,973 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf55933fe6b0fe958 with lease ID 0x3ad51377744934fb: Processing first storage report for DS-9ad9db38-6702-4f38-a084-67817788a15f from datanode DatanodeRegistration(127.0.0.1:46187, datanodeUuid=503b6869-be40-49d2-984d-e9a7c62761c2, infoPort=33999, infoSecurePort=0, ipcPort=42219, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666)
2024-11-24T23:37:47,973 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf55933fe6b0fe958 with lease ID 0x3ad51377744934fb: from storage DS-9ad9db38-6702-4f38-a084-67817788a15f node DatanodeRegistration(127.0.0.1:46187, datanodeUuid=503b6869-be40-49d2-984d-e9a7c62761c2, infoPort=33999, infoSecurePort=0, ipcPort=42219, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:37:48,136 WARN  [Thread-1205 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data3/current/BP-1556390239-172.17.0.3-1732491465666/current, will proceed with Du for space computation calculation, 
2024-11-24T23:37:48,137 WARN  [Thread-1206 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data4/current/BP-1556390239-172.17.0.3-1732491465666/current, will proceed with Du for space computation calculation, 
2024-11-24T23:37:48,161 WARN  [Thread-1181 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:37:48,169 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc520f62539c5ecf3 with lease ID 0x3ad51377744934fc: Processing first storage report for DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77 from datanode DatanodeRegistration(127.0.0.1:33807, datanodeUuid=73182778-e1ce-4aaa-9ee7-4b3063186b65, infoPort=40867, infoSecurePort=0, ipcPort=35073, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666)
2024-11-24T23:37:48,169 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc520f62539c5ecf3 with lease ID 0x3ad51377744934fc: from storage DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77 node DatanodeRegistration(127.0.0.1:33807, datanodeUuid=73182778-e1ce-4aaa-9ee7-4b3063186b65, infoPort=40867, infoSecurePort=0, ipcPort=35073, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T23:37:48,169 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc520f62539c5ecf3 with lease ID 0x3ad51377744934fc: Processing first storage report for DS-4634a6e1-7f0b-4ddb-881d-a2ffa742b702 from datanode DatanodeRegistration(127.0.0.1:33807, datanodeUuid=73182778-e1ce-4aaa-9ee7-4b3063186b65, infoPort=40867, infoSecurePort=0, ipcPort=35073, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666)
2024-11-24T23:37:48,169 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc520f62539c5ecf3 with lease ID 0x3ad51377744934fc: from storage DS-4634a6e1-7f0b-4ddb-881d-a2ffa742b702 node DatanodeRegistration(127.0.0.1:33807, datanodeUuid=73182778-e1ce-4aaa-9ee7-4b3063186b65, infoPort=40867, infoSecurePort=0, ipcPort=35073, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:37:48,245 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261
2024-11-24T23:37:48,248 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/zookeeper_0, clientPort=61696, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T23:37:48,249 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61696
2024-11-24T23:37:48,249 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:37:48,250 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:37:48,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33807 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:37:48,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46187 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:37:48,265 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd with version=8
2024-11-24T23:37:48,265 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/hbase-staging
2024-11-24T23:37:48,267 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:37:48,267 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:37:48,267 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:37:48,267 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:37:48,267 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:37:48,267 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:37:48,267 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T23:37:48,267 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:37:48,268 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43305
2024-11-24T23:37:48,269 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43305 connecting to ZooKeeper ensemble=127.0.0.1:61696
2024-11-24T23:37:48,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:433050x0, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:37:48,336 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43305-0x1016f35d9820000 connected
2024-11-24T23:37:48,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:48,424 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:37:48,426 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:37:48,428 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:37:48,428 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd, hbase.cluster.distributed=false
2024-11-24T23:37:48,430 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:37:48,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43305
2024-11-24T23:37:48,433 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43305
2024-11-24T23:37:48,433 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43305
2024-11-24T23:37:48,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43305
2024-11-24T23:37:48,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43305
2024-11-24T23:37:48,451 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:37:48,451 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:37:48,451 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:37:48,451 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:37:48,451 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:37:48,451 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:37:48,452 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T23:37:48,452 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:37:48,452 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36361
2024-11-24T23:37:48,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:48,454 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36361 connecting to ZooKeeper ensemble=127.0.0.1:61696
2024-11-24T23:37:48,455 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:37:48,456 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:37:48,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363610x0, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:37:48,467 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36361-0x1016f35d9820001 connected
2024-11-24T23:37:48,467 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:37:48,467 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T23:37:48,468 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T23:37:48,468 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T23:37:48,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:37:48,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36361
2024-11-24T23:37:48,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36361
2024-11-24T23:37:48,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36361
2024-11-24T23:37:48,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36361
2024-11-24T23:37:48,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36361
2024-11-24T23:37:48,480 DEBUG [M:0;ba7f32532130:43305 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ba7f32532130:43305
2024-11-24T23:37:48,481 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ba7f32532130,43305,1732491468267
2024-11-24T23:37:48,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:37:48,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:37:48,488 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ba7f32532130,43305,1732491468267
2024-11-24T23:37:48,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:48,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T23:37:48,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:48,498 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T23:37:48,499 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ba7f32532130,43305,1732491468267 from backup master directory
2024-11-24T23:37:48,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:37:48,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ba7f32532130,43305,1732491468267
2024-11-24T23:37:48,508 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:37:48,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:37:48,508 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ba7f32532130,43305,1732491468267
2024-11-24T23:37:48,512 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/hbase.id] with ID: d2b31ccc-926e-40f4-a86b-5a2b6417d3ea
2024-11-24T23:37:48,512 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/.tmp/hbase.id
2024-11-24T23:37:48,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33807 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:37:48,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46187 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:37:48,518 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/.tmp/hbase.id]:[hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/hbase.id]
2024-11-24T23:37:48,529 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:37:48,530 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T23:37:48,531 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-24T23:37:48,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:48,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:48,547 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33807 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:37:48,547 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46187 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:37:48,548 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:37:48,549 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T23:37:48,549 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:37:48,557 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33807 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:37:48,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46187 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:37:48,559 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store
2024-11-24T23:37:48,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46187 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:37:48,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33807 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:37:48,568 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:37:48,568 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:37:48,568 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:37:48,568 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:37:48,568 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:37:48,568 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:37:48,568 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:37:48,568 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491468568Disabling compacts and flushes for region at 1732491468568Disabling writes for close at 1732491468568Writing region close event to WAL at 1732491468568Closed at 1732491468568
2024-11-24T23:37:48,569 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/.initializing
2024-11-24T23:37:48,569 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267
2024-11-24T23:37:48,571 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C43305%2C1732491468267, suffix=, logDir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267, archiveDir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/oldWALs, maxLogs=10
2024-11-24T23:37:48,572 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C43305%2C1732491468267.1732491468572
2024-11-24T23:37:48,577 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267/ba7f32532130%2C43305%2C1732491468267.1732491468572
2024-11-24T23:37:48,582 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40867:40867),(127.0.0.1/127.0.0.1:33999:33999)]
2024-11-24T23:37:48,585 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:37:48,586 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:37:48,586 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:37:48,586 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:37:48,588 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:37:48,589 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T23:37:48,589 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:48,590 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:37:48,590 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:37:48,591 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T23:37:48,591 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:48,591 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:37:48,591 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:37:48,592 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T23:37:48,593 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:48,593 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:37:48,593 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:37:48,594 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T23:37:48,594 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:48,595 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:37:48,595 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:37:48,596 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:37:48,596 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:37:48,597 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:37:48,597 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:37:48,598 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T23:37:48,599 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:37:48,601 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:37:48,602 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=839825, jitterRate=0.06789393723011017}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T23:37:48,603 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732491468586Initializing all the Stores at 1732491468587 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491468587Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491468588 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491468588Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491468588Cleaning up temporary data from old regions at 1732491468597 (+9 ms)Region opened successfully at 1732491468602 (+5 ms)
2024-11-24T23:37:48,604 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T23:37:48,607 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cad24ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:37:48,608 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T23:37:48,608 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T23:37:48,608 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T23:37:48,608 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T23:37:48,609 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T23:37:48,609 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T23:37:48,609 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T23:37:48,611 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T23:37:48,613 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T23:37:48,624 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T23:37:48,625 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T23:37:48,626 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T23:37:48,634 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T23:37:48,635 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T23:37:48,636 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T23:37:48,645 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T23:37:48,647 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T23:37:48,656 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T23:37:48,660 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T23:37:48,666 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T23:37:48,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:37:48,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:48,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:37:48,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:48,678 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ba7f32532130,43305,1732491468267, sessionid=0x1016f35d9820000, setting cluster-up flag (Was=false)
2024-11-24T23:37:48,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:48,731 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T23:37:48,732 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,43305,1732491468267
2024-11-24T23:37:48,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:48,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:48,782 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T23:37:48,784 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,43305,1732491468267
2024-11-24T23:37:48,785 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T23:37:48,787 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T23:37:48,787 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T23:37:48,788 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T23:37:48,788 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ba7f32532130,43305,1732491468267
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T23:37:48,789 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:37:48,789 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:37:48,789 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:37:48,789 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:37:48,789 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ba7f32532130:0, corePoolSize=10, maxPoolSize=10
2024-11-24T23:37:48,790 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,790 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:37:48,790 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,791 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732491498791
2024-11-24T23:37:48,791 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T23:37:48,791 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T23:37:48,791 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T23:37:48,791 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T23:37:48,791 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T23:37:48,791 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T23:37:48,792 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:37:48,792 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T23:37:48,793 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:48,793 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T23:37:48,796 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,797 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T23:37:48,797 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T23:37:48,797 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T23:37:48,798 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T23:37:48,798 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T23:37:48,800 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491468798,5,FailOnTimeoutGroup]
2024-11-24T23:37:48,803 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491468800,5,FailOnTimeoutGroup]
2024-11-24T23:37:48,803 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,804 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T23:37:48,804 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,804 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46187 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:37:48,807 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33807 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:37:48,808 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T23:37:48,808 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd
2024-11-24T23:37:48,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46187 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:37:48,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33807 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:37:48,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:37:48,824 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:37:48,826 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:37:48,826 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:48,827 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:37:48,827 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:37:48,828 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:37:48,828 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:48,829 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:37:48,829 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:37:48,831 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:37:48,831 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:48,831 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:37:48,831 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:37:48,833 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:37:48,833 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:48,833 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:37:48,833 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:37:48,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740
2024-11-24T23:37:48,835 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740
2024-11-24T23:37:48,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:37:48,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:37:48,837 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:37:48,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:37:48,841 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:37:48,842 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=824236, jitterRate=0.04807044565677643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:37:48,842 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732491468822Initializing all the Stores at 1732491468823 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491468823Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491468824 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491468824Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491468824Cleaning up temporary data from old regions at 1732491468836 (+12 ms)Region opened successfully at 1732491468842 (+6 ms)
2024-11-24T23:37:48,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:37:48,843 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:37:48,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:37:48,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:37:48,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:37:48,843 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:37:48,844 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491468843Disabling compacts and flushes for region at 1732491468843Disabling writes for close at 1732491468843Writing region close event to WAL at 1732491468843Closed at 1732491468843
2024-11-24T23:37:48,845 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:37:48,845 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T23:37:48,846 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T23:37:48,848 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:37:48,851 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T23:37:48,872 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(746): ClusterId : d2b31ccc-926e-40f4-a86b-5a2b6417d3ea
2024-11-24T23:37:48,872 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T23:37:48,878 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T23:37:48,878 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T23:37:48,888 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T23:37:48,889 DEBUG [RS:0;ba7f32532130:36361 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6248c22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:37:48,901 DEBUG [RS:0;ba7f32532130:36361 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ba7f32532130:36361
2024-11-24T23:37:48,901 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T23:37:48,901 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T23:37:48,901 DEBUG [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T23:37:48,902 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba7f32532130,43305,1732491468267 with port=36361, startcode=1732491468451
2024-11-24T23:37:48,902 DEBUG [RS:0;ba7f32532130:36361 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T23:37:48,904 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47777, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T23:37:48,905 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43305 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba7f32532130,36361,1732491468451
2024-11-24T23:37:48,905 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43305 {}] master.ServerManager(517): Registering regionserver=ba7f32532130,36361,1732491468451
2024-11-24T23:37:48,907 DEBUG [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd
2024-11-24T23:37:48,907 DEBUG [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45999
2024-11-24T23:37:48,907 DEBUG [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T23:37:48,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:37:48,919 DEBUG [RS:0;ba7f32532130:36361 {}] zookeeper.ZKUtil(111): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba7f32532130,36361,1732491468451
2024-11-24T23:37:48,920 WARN  [RS:0;ba7f32532130:36361 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:37:48,920 INFO  [RS:0;ba7f32532130:36361 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:37:48,920 DEBUG [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451
2024-11-24T23:37:48,920 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba7f32532130,36361,1732491468451]
2024-11-24T23:37:48,924 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T23:37:48,926 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T23:37:48,926 INFO  [RS:0;ba7f32532130:36361 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T23:37:48,926 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,927 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T23:37:48,928 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T23:37:48,928 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,928 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,928 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,928 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,928 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,928 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,928 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:37:48,929 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,929 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,929 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,929 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,929 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,929 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:37:48,929 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:37:48,929 DEBUG [RS:0;ba7f32532130:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:37:48,932 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,932 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,933 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,933 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,933 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,933 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,36361,1732491468451-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:37:48,948 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T23:37:48,949 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,36361,1732491468451-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,949 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,949 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.Replication(171): ba7f32532130,36361,1732491468451 started
2024-11-24T23:37:48,965 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:48,965 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(1482): Serving as ba7f32532130,36361,1732491468451, RpcServer on ba7f32532130/172.17.0.3:36361, sessionid=0x1016f35d9820001
2024-11-24T23:37:48,965 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T23:37:48,965 DEBUG [RS:0;ba7f32532130:36361 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba7f32532130,36361,1732491468451
2024-11-24T23:37:48,965 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,36361,1732491468451'
2024-11-24T23:37:48,965 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T23:37:48,966 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T23:37:48,966 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T23:37:48,966 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T23:37:48,966 DEBUG [RS:0;ba7f32532130:36361 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba7f32532130,36361,1732491468451
2024-11-24T23:37:48,966 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,36361,1732491468451'
2024-11-24T23:37:48,966 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T23:37:48,967 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T23:37:48,967 DEBUG [RS:0;ba7f32532130:36361 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T23:37:48,967 INFO  [RS:0;ba7f32532130:36361 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T23:37:48,967 INFO  [RS:0;ba7f32532130:36361 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T23:37:49,001 WARN  [ba7f32532130:43305 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T23:37:49,069 INFO  [RS:0;ba7f32532130:36361 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C36361%2C1732491468451, suffix=, logDir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451, archiveDir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/oldWALs, maxLogs=32
2024-11-24T23:37:49,070 INFO  [RS:0;ba7f32532130:36361 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C36361%2C1732491468451.1732491469070
2024-11-24T23:37:49,077 INFO  [RS:0;ba7f32532130:36361 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070
2024-11-24T23:37:49,084 DEBUG [RS:0;ba7f32532130:36361 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40867:40867),(127.0.0.1/127.0.0.1:33999:33999)]
2024-11-24T23:37:49,251 DEBUG [ba7f32532130:43305 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T23:37:49,252 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba7f32532130,36361,1732491468451
2024-11-24T23:37:49,253 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,36361,1732491468451, state=OPENING
2024-11-24T23:37:49,395 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:49,430 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T23:37:49,453 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:49,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:49,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:37:49,508 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:37:49,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:37:49,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:37:49,508 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,36361,1732491468451}]
2024-11-24T23:37:49,661 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T23:37:49,663 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57841, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T23:37:49,666 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T23:37:49,666 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:37:49,668 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C36361%2C1732491468451.meta, suffix=.meta, logDir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451, archiveDir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/oldWALs, maxLogs=32
2024-11-24T23:37:49,669 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C36361%2C1732491468451.meta.1732491469669.meta
2024-11-24T23:37:49,674 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.meta.1732491469669.meta
2024-11-24T23:37:49,675 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40867:40867),(127.0.0.1/127.0.0.1:33999:33999)]
2024-11-24T23:37:49,675 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:37:49,676 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T23:37:49,676 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T23:37:49,676 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T23:37:49,676 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T23:37:49,676 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:37:49,676 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T23:37:49,676 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T23:37:49,678 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:37:49,679 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:37:49,679 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:49,679 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:37:49,679 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:37:49,680 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:37:49,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:49,681 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:37:49,681 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:37:49,682 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:37:49,682 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:49,682 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:37:49,682 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:37:49,683 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:37:49,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:49,684 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:37:49,684 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:37:49,684 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740
2024-11-24T23:37:49,686 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740
2024-11-24T23:37:49,687 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:37:49,687 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:37:49,688 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:37:49,689 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:37:49,689 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=779979, jitterRate=-0.008205518126487732}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:37:49,689 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T23:37:49,690 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732491469676Writing region info on filesystem at 1732491469676Initializing all the Stores at 1732491469677 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491469677Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491469678 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491469678Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491469678Cleaning up temporary data from old regions at 1732491469687 (+9 ms)Running coprocessor post-open hooks at 1732491469689 (+2 ms)Region opened successfully at 1732491469690 (+1 ms)
2024-11-24T23:37:49,691 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732491469661
2024-11-24T23:37:49,693 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T23:37:49,693 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T23:37:49,694 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,36361,1732491468451
2024-11-24T23:37:49,695 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,36361,1732491468451, state=OPEN
2024-11-24T23:37:49,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:37:49,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:37:49,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:37:49,828 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba7f32532130,36361,1732491468451
2024-11-24T23:37:49,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:37:49,832 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T23:37:49,832 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,36361,1732491468451 in 320 msec
2024-11-24T23:37:49,835 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T23:37:49,835 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 987 msec
2024-11-24T23:37:49,836 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:37:49,836 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T23:37:49,838 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:37:49,838 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36361,1732491468451, seqNum=-1]
2024-11-24T23:37:49,838 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:37:49,840 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53083, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:37:49,847 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0580 sec
2024-11-24T23:37:49,847 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732491469847, completionTime=-1
2024-11-24T23:37:49,847 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T23:37:49,847 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T23:37:49,849 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T23:37:49,849 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732491529849
2024-11-24T23:37:49,849 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732491589849
2024-11-24T23:37:49,849 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-24T23:37:49,849 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,43305,1732491468267-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:49,849 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,43305,1732491468267-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:49,850 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,43305,1732491468267-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:49,850 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ba7f32532130:43305, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:49,850 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:49,850 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:49,852 DEBUG [master/ba7f32532130:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T23:37:49,855 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.346sec
2024-11-24T23:37:49,855 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T23:37:49,855 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T23:37:49,855 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T23:37:49,855 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T23:37:49,855 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T23:37:49,855 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,43305,1732491468267-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:37:49,855 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,43305,1732491468267-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T23:37:49,858 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T23:37:49,858 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T23:37:49,858 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,43305,1732491468267-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:37:49,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343c572e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:37:49,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ba7f32532130,43305,-1 for getting cluster id
2024-11-24T23:37:49,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T23:37:49,875 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd2b31ccc-926e-40f4-a86b-5a2b6417d3ea'
2024-11-24T23:37:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T23:37:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d2b31ccc-926e-40f4-a86b-5a2b6417d3ea"

2024-11-24T23:37:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b5694a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:37:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ba7f32532130,43305,-1]
2024-11-24T23:37:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T23:37:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:37:49,877 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T23:37:49,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3d6246, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:37:49,879 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:37:49,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,36361,1732491468451, seqNum=-1]
2024-11-24T23:37:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:37:49,881 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:37:49,883 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ba7f32532130,43305,1732491468267
2024-11-24T23:37:49,883 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:37:49,886 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T23:37:49,886 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-24T23:37:49,886 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-24T23:37:49,886 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-24T23:37:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is ba7f32532130,43305,1732491468267
2024-11-24T23:37:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23fdc080
2024-11-24T23:37:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-24T23:37:49,889 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-24T23:37:49,890 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43305 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-24T23:37:49,890 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43305 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-24T23:37:49,890 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43305 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:37:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43305 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-24T23:37:49,893 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-24T23:37:49,893 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:49,893 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43305 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-24T23:37:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43305 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T23:37:49,894 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-24T23:37:49,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33807 is added to blk_1073741835_1011 (size=395)
2024-11-24T23:37:49,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46187 is added to blk_1073741835_1011 (size=395)
2024-11-24T23:37:49,903 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dce5568d03590a86e4e8225b88370f8d, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd
2024-11-24T23:37:49,908 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46187 is added to blk_1073741836_1012 (size=78)
2024-11-24T23:37:49,908 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33807 is added to blk_1073741836_1012 (size=78)
2024-11-24T23:37:49,909 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:37:49,909 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing dce5568d03590a86e4e8225b88370f8d, disabling compactions & flushes
2024-11-24T23:37:49,909 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:37:49,909 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:37:49,909 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d. after waiting 0 ms
2024-11-24T23:37:49,909 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:37:49,909 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:37:49,909 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for dce5568d03590a86e4e8225b88370f8d:
Waiting for close lock at 1732491469909Disabling compacts and flushes for region at 1732491469909Disabling writes for close at 1732491469909Writing region close event to WAL at 1732491469909Closed at 1732491469909
2024-11-24T23:37:49,911 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-24T23:37:49,911 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1732491469911"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732491469911"}]},"ts":"1732491469911"}
2024-11-24T23:37:49,913 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-24T23:37:49,914 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-24T23:37:49,915 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732491469914"}]},"ts":"1732491469914"}
2024-11-24T23:37:49,917 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-24T23:37:49,918 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=dce5568d03590a86e4e8225b88370f8d, ASSIGN}]
2024-11-24T23:37:49,919 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=dce5568d03590a86e4e8225b88370f8d, ASSIGN
2024-11-24T23:37:49,920 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=dce5568d03590a86e4e8225b88370f8d, ASSIGN; state=OFFLINE, location=ba7f32532130,36361,1732491468451; forceNewPlan=false, retain=false
2024-11-24T23:37:50,071 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dce5568d03590a86e4e8225b88370f8d, regionState=OPENING, regionLocation=ba7f32532130,36361,1732491468451
2024-11-24T23:37:50,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=dce5568d03590a86e4e8225b88370f8d, ASSIGN because future has completed
2024-11-24T23:37:50,075 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dce5568d03590a86e4e8225b88370f8d, server=ba7f32532130,36361,1732491468451}]
2024-11-24T23:37:50,233 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:37:50,233 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => dce5568d03590a86e4e8225b88370f8d, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:37:50,234 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:37:50,234 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:37:50,234 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:37:50,234 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:37:50,235 INFO  [StoreOpener-dce5568d03590a86e4e8225b88370f8d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region dce5568d03590a86e4e8225b88370f8d 
2024-11-24T23:37:50,237 INFO  [StoreOpener-dce5568d03590a86e4e8225b88370f8d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dce5568d03590a86e4e8225b88370f8d columnFamilyName info
2024-11-24T23:37:50,237 DEBUG [StoreOpener-dce5568d03590a86e4e8225b88370f8d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:37:50,237 INFO  [StoreOpener-dce5568d03590a86e4e8225b88370f8d-1 {}] regionserver.HStore(327): Store=dce5568d03590a86e4e8225b88370f8d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:37:50,238 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:37:50,239 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/default/TestLogRolling-testLogRollOnPipelineRestart/dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:37:50,239 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/default/TestLogRolling-testLogRollOnPipelineRestart/dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:37:50,240 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:37:50,240 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:37:50,242 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:37:50,244 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/default/TestLogRolling-testLogRollOnPipelineRestart/dce5568d03590a86e4e8225b88370f8d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:37:50,245 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened dce5568d03590a86e4e8225b88370f8d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=825480, jitterRate=0.049653083086013794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T23:37:50,245 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:37:50,245 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for dce5568d03590a86e4e8225b88370f8d:
Running coprocessor pre-open hook at 1732491470234Writing region info on filesystem at 1732491470234Initializing all the Stores at 1732491470235 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491470235Cleaning up temporary data from old regions at 1732491470240 (+5 ms)Running coprocessor post-open hooks at 1732491470245 (+5 ms)Region opened successfully at 1732491470245
2024-11-24T23:37:50,246 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d., pid=6, masterSystemTime=1732491470228
2024-11-24T23:37:50,249 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:37:50,249 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:37:50,250 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dce5568d03590a86e4e8225b88370f8d, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,36361,1732491468451
2024-11-24T23:37:50,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dce5568d03590a86e4e8225b88370f8d, server=ba7f32532130,36361,1732491468451 because future has completed
2024-11-24T23:37:50,257 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-24T23:37:50,257 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dce5568d03590a86e4e8225b88370f8d, server=ba7f32532130,36361,1732491468451 in 179 msec
2024-11-24T23:37:50,260 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-24T23:37:50,260 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=dce5568d03590a86e4e8225b88370f8d, ASSIGN in 339 msec
2024-11-24T23:37:50,261 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-24T23:37:50,261 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732491470261"}]},"ts":"1732491470261"}
2024-11-24T23:37:50,264 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-24T23:37:50,265 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-24T23:37:50,268 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 375 msec
2024-11-24T23:37:50,396 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:50,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:51,397 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:51,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:52,398 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:52,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:53,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T23:37:53,144 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-24T23:37:53,145 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-24T23:37:53,145 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-24T23:37:53,146 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:37:53,146 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-24T23:37:53,399 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:53,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:54,399 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:54,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:54,696 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:54,696 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:54,697 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:54,697 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:54,698 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:54,699 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:54,704 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:54,705 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:54,705 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:54,708 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:55,215 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T23:37:55,244 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:55,244 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:55,245 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:55,246 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:55,246 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:55,246 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:55,250 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:55,250 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:55,250 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:55,253 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:37:55,258 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-24T23:37:55,258 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-24T23:37:55,400 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:55,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:56,401 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:56,458 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:57,402 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:57,459 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:58,402 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:58,459 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:59,403 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:59,460 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:37:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43305 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T23:37:59,922 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-24T23:37:59,922 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-24T23:37:59,925 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-24T23:37:59,925 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:37:59,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d., hostname=ba7f32532130,36361,1732491468451, seqNum=2]
2024-11-24T23:38:00,404 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:00,461 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:01,405 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:01,462 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:01,932 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070
2024-11-24T23:38:01,932 WARN  [ResponseProcessor for block BP-1556390239-172.17.0.3-1732491465666:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556390239-172.17.0.3-1732491465666:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:01,932 WARN  [ResponseProcessor for block BP-1556390239-172.17.0.3-1732491465666:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556390239-172.17.0.3-1732491465666:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:01,932 WARN  [ResponseProcessor for block BP-1556390239-172.17.0.3-1732491465666:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556390239-172.17.0.3-1732491465666:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:01,933 WARN  [DataStreamer for file /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.meta.1732491469669.meta block BP-1556390239-172.17.0.3-1732491465666:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1556390239-172.17.0.3-1732491465666:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33807,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK], DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33807,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK]) is bad.
2024-11-24T23:38:01,933 WARN  [DataStreamer for file /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267/ba7f32532130%2C43305%2C1732491468267.1732491468572 block BP-1556390239-172.17.0.3-1732491465666:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1556390239-172.17.0.3-1732491465666:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33807,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK], DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33807,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK]) is bad.
2024-11-24T23:38:01,933 WARN  [DataStreamer for file /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070 block BP-1556390239-172.17.0.3-1732491465666:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1556390239-172.17.0.3-1732491465666:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33807,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK], DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33807,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK]) is bad.
2024-11-24T23:38:01,933 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1552087297_22 at /127.0.0.1:48210 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33807:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48210 dst: /127.0.0.1:33807
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:01,933 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1552087297_22 at /127.0.0.1:48206 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33807:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48206 dst: /127.0.0.1:33807
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:01,934 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1130521888_22 at /127.0.0.1:49086 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49086 dst: /127.0.0.1:46187
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:01,934 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1552087297_22 at /127.0.0.1:49110 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49110 dst: /127.0.0.1:46187
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:01,934 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1552087297_22 at /127.0.0.1:49118 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49118 dst: /127.0.0.1:46187
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:01,933 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1130521888_22 at /127.0.0.1:48182 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33807:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48182 dst: /127.0.0.1:33807
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:02,069 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@716d71e9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:02,069 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@60deb4a2{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:38:02,069 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:38:02,070 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@59ddfe6e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:38:02,070 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41d5af00{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,STOPPED}
2024-11-24T23:38:02,071 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:38:02,071 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:38:02,071 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556390239-172.17.0.3-1732491465666 (Datanode Uuid 73182778-e1ce-4aaa-9ee7-4b3063186b65) service to localhost/127.0.0.1:45999
2024-11-24T23:38:02,071 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:38:02,072 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data3/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:02,073 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data4/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:02,073 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:38:02,083 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:38:02,087 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:38:02,088 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:38:02,088 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:38:02,089 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:38:02,089 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@717499fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:38:02,090 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62802d30{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:38:02,192 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1dcfabbb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/java.io.tmpdir/jetty-localhost-45979-hadoop-hdfs-3_4_1-tests_jar-_-any-13905190684934435865/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:02,192 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6614137b{HTTP/1.1, (http/1.1)}{localhost:45979}
2024-11-24T23:38:02,193 INFO  [Time-limited test {}] server.Server(415): Started @176776ms
2024-11-24T23:38:02,194 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:38:02,210 WARN  [ResponseProcessor for block BP-1556390239-172.17.0.3-1732491465666:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556390239-172.17.0.3-1732491465666:blk_1073741834_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:02,210 WARN  [ResponseProcessor for block BP-1556390239-172.17.0.3-1732491465666:blk_1073741833_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556390239-172.17.0.3-1732491465666:blk_1073741833_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:02,210 WARN  [ResponseProcessor for block BP-1556390239-172.17.0.3-1732491465666:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556390239-172.17.0.3-1732491465666:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:02,210 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1552087297_22 at /127.0.0.1:35950 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35950 dst: /127.0.0.1:46187
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:02,210 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1130521888_22 at /127.0.0.1:35936 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35936 dst: /127.0.0.1:46187
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:02,211 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1552087297_22 at /127.0.0.1:35934 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35934 dst: /127.0.0.1:46187
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:02,213 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@51031d29{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:02,213 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1a48749e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:38:02,213 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:38:02,213 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3bfb2c17{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:38:02,213 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a7cb65f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,STOPPED}
2024-11-24T23:38:02,214 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:38:02,214 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:38:02,215 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:38:02,215 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556390239-172.17.0.3-1732491465666 (Datanode Uuid 503b6869-be40-49d2-984d-e9a7c62761c2) service to localhost/127.0.0.1:45999
2024-11-24T23:38:02,215 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data1/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:02,215 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data2/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:02,215 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:38:02,226 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:38:02,231 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:38:02,236 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:38:02,236 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:38:02,236 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:38:02,237 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3146549d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:38:02,237 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6baabd83{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:38:02,333 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21404da7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/java.io.tmpdir/jetty-localhost-39539-hadoop-hdfs-3_4_1-tests_jar-_-any-9363197374176115415/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:02,333 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e8914c8{HTTP/1.1, (http/1.1)}{localhost:39539}
2024-11-24T23:38:02,333 INFO  [Time-limited test {}] server.Server(415): Started @176917ms
2024-11-24T23:38:02,334 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:38:02,405 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:02,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:02,856 WARN  [Thread-1329 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:38:02,860 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3182c88b9dd1ecb7 with lease ID 0x3ad51377744934fd: from storage DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77 node DatanodeRegistration(127.0.0.1:37989, datanodeUuid=73182778-e1ce-4aaa-9ee7-4b3063186b65, infoPort=45547, infoSecurePort=0, ipcPort=46737, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T23:38:02,860 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3182c88b9dd1ecb7 with lease ID 0x3ad51377744934fd: from storage DS-4634a6e1-7f0b-4ddb-881d-a2ffa742b702 node DatanodeRegistration(127.0.0.1:37989, datanodeUuid=73182778-e1ce-4aaa-9ee7-4b3063186b65, infoPort=45547, infoSecurePort=0, ipcPort=46737, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:38:02,991 WARN  [Thread-1349 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:38:02,994 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xce5f072ab825d410 with lease ID 0x3ad51377744934fe: from storage DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13 node DatanodeRegistration(127.0.0.1:33343, datanodeUuid=503b6869-be40-49d2-984d-e9a7c62761c2, infoPort=34855, infoSecurePort=0, ipcPort=43917, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:38:02,994 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xce5f072ab825d410 with lease ID 0x3ad51377744934fe: from storage DS-9ad9db38-6702-4f38-a084-67817788a15f node DatanodeRegistration(127.0.0.1:33343, datanodeUuid=503b6869-be40-49d2-984d-e9a7c62761c2, infoPort=34855, infoSecurePort=0, ipcPort=43917, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:38:03,352 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-24T23:38:03,355 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-24T23:38:03,356 ERROR [FSHLog-0-hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd-prefix:ba7f32532130,36361,1732491468451 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:03,357 WARN  [FSHLog-0-hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd-prefix:ba7f32532130,36361,1732491468451 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:03,357 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C36361%2C1732491468451:(num 1732491469070) roll requested
2024-11-24T23:38:03,357 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C36361%2C1732491468451.1732491483357
2024-11-24T23:38:03,364 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070 newFile=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357
2024-11-24T23:38:03,364 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:03,364 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:03,364 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:03,365 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:03,365 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:03,365 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357
2024-11-24T23:38:03,365 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:03,366 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:03,366 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070
2024-11-24T23:38:03,366 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34855:34855),(127.0.0.1/127.0.0.1:45547:45547)]
2024-11-24T23:38:03,366 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070 is not closed yet, will try archiving it next time
2024-11-24T23:38:03,366 WARN  [IPC Server handler 0 on default port 45999 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1013
2024-11-24T23:38:03,367 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070 after 1ms
2024-11-24T23:38:03,406 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:03,464 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:04,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:04,465 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:04,858 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1013: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-24T23:38:05,370 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-24T23:38:05,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:05,465 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:06,408 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:06,466 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:07,367 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070 after 4001ms
2024-11-24T23:38:07,375 WARN  [ResponseProcessor for block BP-1556390239-172.17.0.3-1732491465666:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556390239-172.17.0.3-1732491465666:blk_1073741837_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:07,375 WARN  [DataStreamer for file /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357 block BP-1556390239-172.17.0.3-1732491465666:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1556390239-172.17.0.3-1732491465666:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33343,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK], DatanodeInfoWithStorage[127.0.0.1:37989,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33343,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]) is bad.
2024-11-24T23:38:07,376 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1552087297_22 at /127.0.0.1:40258 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:37989:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40258 dst: /127.0.0.1:37989
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:07,375 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1552087297_22 at /127.0.0.1:41032 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:33343:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41032 dst: /127.0.0.1:33343
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:07,408 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:07,427 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21404da7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:07,427 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e8914c8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:38:07,427 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:38:07,427 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6baabd83{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:38:07,427 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3146549d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,STOPPED}
2024-11-24T23:38:07,428 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:38:07,428 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:38:07,428 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:38:07,428 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556390239-172.17.0.3-1732491465666 (Datanode Uuid 503b6869-be40-49d2-984d-e9a7c62761c2) service to localhost/127.0.0.1:45999
2024-11-24T23:38:07,429 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data1/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:07,429 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data2/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:07,429 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:38:07,437 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:38:07,440 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:38:07,441 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:38:07,441 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:38:07,441 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:38:07,441 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@518d8d55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:38:07,442 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22abc971{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:38:07,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:07,537 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@77a301c7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/java.io.tmpdir/jetty-localhost-41867-hadoop-hdfs-3_4_1-tests_jar-_-any-4537475442014136929/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:07,538 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7c0d41d9{HTTP/1.1, (http/1.1)}{localhost:41867}
2024-11-24T23:38:07,538 INFO  [Time-limited test {}] server.Server(415): Started @182122ms
2024-11-24T23:38:07,539 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:38:07,558 WARN  [ResponseProcessor for block BP-1556390239-172.17.0.3-1732491465666:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556390239-172.17.0.3-1732491465666:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:07,558 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1552087297_22 at /127.0.0.1:40290 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:37989:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40290 dst: /127.0.0.1:37989
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:07,560 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1dcfabbb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:07,560 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6614137b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:38:07,560 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:38:07,561 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62802d30{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:38:07,561 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@717499fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,STOPPED}
2024-11-24T23:38:07,562 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:38:07,562 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:38:07,562 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:38:07,562 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556390239-172.17.0.3-1732491465666 (Datanode Uuid 73182778-e1ce-4aaa-9ee7-4b3063186b65) service to localhost/127.0.0.1:45999
2024-11-24T23:38:07,563 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data3/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:07,563 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data4/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:07,563 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:38:07,578 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:38:07,581 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:38:07,584 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:38:07,584 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:38:07,584 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:38:07,584 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bc7279c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:38:07,584 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3df3f65e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:38:07,680 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f03ce36{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/java.io.tmpdir/jetty-localhost-39823-hadoop-hdfs-3_4_1-tests_jar-_-any-1943162689430757673/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:07,680 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@cbccde7{HTTP/1.1, (http/1.1)}{localhost:39823}
2024-11-24T23:38:07,680 INFO  [Time-limited test {}] server.Server(415): Started @182264ms
2024-11-24T23:38:07,682 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:38:08,053 WARN  [Thread-1403 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:38:08,056 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97753e526556a7e9 with lease ID 0x3ad51377744934ff: from storage DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13 node DatanodeRegistration(127.0.0.1:38923, datanodeUuid=503b6869-be40-49d2-984d-e9a7c62761c2, infoPort=41107, infoSecurePort=0, ipcPort=42483, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:38:08,056 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97753e526556a7e9 with lease ID 0x3ad51377744934ff: from storage DS-9ad9db38-6702-4f38-a084-67817788a15f node DatanodeRegistration(127.0.0.1:38923, datanodeUuid=503b6869-be40-49d2-984d-e9a7c62761c2, infoPort=41107, infoSecurePort=0, ipcPort=42483, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:38:08,208 WARN  [Thread-1423 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:38:08,210 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd26ed06a135ea996 with lease ID 0x3ad5137774493500: from storage DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77 node DatanodeRegistration(127.0.0.1:42057, datanodeUuid=73182778-e1ce-4aaa-9ee7-4b3063186b65, infoPort=40201, infoSecurePort=0, ipcPort=42983, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:38:08,211 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd26ed06a135ea996 with lease ID 0x3ad5137774493500: from storage DS-4634a6e1-7f0b-4ddb-881d-a2ffa742b702 node DatanodeRegistration(127.0.0.1:42057, datanodeUuid=73182778-e1ce-4aaa-9ee7-4b3063186b65, infoPort=40201, infoSecurePort=0, ipcPort=42983, storageInfo=lv=-57;cid=testClusterID;nsid=994175259;c=1732491465666), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:38:08,409 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:08,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:08,698 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-24T23:38:08,701 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-24T23:38:08,702 ERROR [FSHLog-0-hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd-prefix:ba7f32532130,36361,1732491468451 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37989,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:08,702 WARN  [FSHLog-0-hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd-prefix:ba7f32532130,36361,1732491468451 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37989,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:08,702 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C36361%2C1732491468451:(num 1732491483357) roll requested
2024-11-24T23:38:08,703 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C36361%2C1732491468451.1732491488703
2024-11-24T23:38:08,711 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357 newFile=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491488703
2024-11-24T23:38:08,711 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:08,711 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:08,711 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:08,711 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:08,711 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:08,711 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491488703
2024-11-24T23:38:08,711 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37989,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:08,712 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37989,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:08,712 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357
2024-11-24T23:38:08,712 WARN  [IPC Server handler 1 on default port 45999 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-24T23:38:08,712 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357 after 0ms
2024-11-24T23:38:08,717 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40201:40201),(127.0.0.1/127.0.0.1:41107:41107)]
2024-11-24T23:38:08,717 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357 is not closed yet, will try archiving it next time
2024-11-24T23:38:09,410 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:09,468 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:10,411 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:10,469 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:10,719 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C36361%2C1732491468451.1732491490718
2024-11-24T23:38:10,724 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491488703 newFile=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718
2024-11-24T23:38:10,724 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:10,724 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:10,724 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:10,724 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:10,724 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:10,725 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491488703 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718
2024-11-24T23:38:10,725 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40201:40201),(127.0.0.1/127.0.0.1:41107:41107)]
2024-11-24T23:38:10,725 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357 is not closed yet, will try archiving it next time
2024-11-24T23:38:10,726 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491488703 is not closed yet, will try archiving it next time
2024-11-24T23:38:10,726 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070
2024-11-24T23:38:10,726 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070
2024-11-24T23:38:10,726 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070 after 0ms
2024-11-24T23:38:10,726 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741838_1019 (size=1264)
2024-11-24T23:38:10,726 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070
2024-11-24T23:38:10,727 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357 is not closed yet, will try archiving it next time
2024-11-24T23:38:10,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741838_1019 (size=1264)
2024-11-24T23:38:10,736 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1732491470246/Put/vlen=218/seqid=0]
2024-11-24T23:38:10,736 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1732491479930/Put/vlen=1045/seqid=0]
2024-11-24T23:38:10,736 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491469070
2024-11-24T23:38:10,736 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357
2024-11-24T23:38:10,736 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357
2024-11-24T23:38:10,737 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357 after 1ms
2024-11-24T23:38:10,737 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357
2024-11-24T23:38:10,740 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1732491483356/Put/vlen=1045/seqid=0]
2024-11-24T23:38:10,740 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1732491485372/Put/vlen=1045/seqid=0]
2024-11-24T23:38:10,740 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357
2024-11-24T23:38:10,740 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491488703
2024-11-24T23:38:10,740 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491488703
2024-11-24T23:38:10,741 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491488703 after 1ms
2024-11-24T23:38:10,741 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491488703
2024-11-24T23:38:10,744 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1732491488702/Put/vlen=1045/seqid=0]
2024-11-24T23:38:10,744 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718
2024-11-24T23:38:10,744 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718
2024-11-24T23:38:10,744 WARN  [IPC Server handler 2 on default port 45999 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-24T23:38:10,745 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718 after 1ms
2024-11-24T23:38:11,055 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-24T23:38:11,213 WARN  [ResponseProcessor for block BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:11,213 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1130521888_22 at /127.0.0.1:52266 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:42057:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52266 dst: /127.0.0.1:42057
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42057 remote=/127.0.0.1:52266]. Total timeout mills is 60000, 59511 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:11,214 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1130521888_22 at /127.0.0.1:59190 [Receiving block BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:38923:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59190 dst: /127.0.0.1:38923
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:11,214 WARN  [DataStreamer for file /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718 block BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42057,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK], DatanodeInfoWithStorage[127.0.0.1:38923,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42057,DS-8ff72d07-36bb-4ca1-ae3f-18787aaeed77,DISK]) is bad.
2024-11-24T23:38:11,216 WARN  [DataStreamer for file /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718 block BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:11,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741839_1022 (size=85)
2024-11-24T23:38:11,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741839_1022 (size=85)
2024-11-24T23:38:11,412 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:11,470 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:12,413 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:12,471 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:12,714 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491483357 after 4002ms
2024-11-24T23:38:13,414 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:13,473 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:14,415 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:14,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:14,746 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718 after 4002ms
2024-11-24T23:38:14,746 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718
2024-11-24T23:38:14,754 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718
2024-11-24T23:38:14,754 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing dce5568d03590a86e4e8225b88370f8d 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-24T23:38:14,755 ERROR [FSHLog-0-hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd-prefix:ba7f32532130,36361,1732491468451 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:14,755 WARN  [FSHLog-0-hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd-prefix:ba7f32532130,36361,1732491468451 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:14,756 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C36361%2C1732491468451:(num 1732491490718) roll requested
2024-11-24T23:38:14,756 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C36361%2C1732491468451.1732491494756
2024-11-24T23:38:14,761 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718 newFile=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491494756
2024-11-24T23:38:14,761 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:14,761 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:14,762 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:14,762 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:14,762 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:14,762 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491494756
2024-11-24T23:38:14,762 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:14,763 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41107:41107),(127.0.0.1/127.0.0.1:40201:40201)]
2024-11-24T23:38:14,763 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718 is not closed yet, will try archiving it next time
2024-11-24T23:38:14,762 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1556390239-172.17.0.3-1732491465666:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:14,763 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718
2024-11-24T23:38:14,763 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718 after 0ms
2024-11-24T23:38:14,764 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.1732491490718 to hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/oldWALs/ba7f32532130%2C36361%2C1732491468451.1732491490718
2024-11-24T23:38:14,780 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/default/TestLogRolling-testLogRollOnPipelineRestart/dce5568d03590a86e4e8225b88370f8d/.tmp/info/eb657c084b4c45429b81ebefe3158008 is 1080, key is row1002/info:/1732491479930/Put/seqid=0
2024-11-24T23:38:14,784 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741841_1024 (size=9270)
2024-11-24T23:38:14,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741841_1024 (size=9270)
2024-11-24T23:38:14,785 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/default/TestLogRolling-testLogRollOnPipelineRestart/dce5568d03590a86e4e8225b88370f8d/.tmp/info/eb657c084b4c45429b81ebefe3158008
2024-11-24T23:38:14,791 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/default/TestLogRolling-testLogRollOnPipelineRestart/dce5568d03590a86e4e8225b88370f8d/.tmp/info/eb657c084b4c45429b81ebefe3158008 as hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/default/TestLogRolling-testLogRollOnPipelineRestart/dce5568d03590a86e4e8225b88370f8d/info/eb657c084b4c45429b81ebefe3158008
2024-11-24T23:38:14,798 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/default/TestLogRolling-testLogRollOnPipelineRestart/dce5568d03590a86e4e8225b88370f8d/info/eb657c084b4c45429b81ebefe3158008, entries=4, sequenceid=8, filesize=9.1 K
2024-11-24T23:38:14,799 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for dce5568d03590a86e4e8225b88370f8d in 45ms, sequenceid=8, compaction requested=false
2024-11-24T23:38:14,799 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for dce5568d03590a86e4e8225b88370f8d:

2024-11-24T23:38:14,799 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-24T23:38:14,799 ERROR [FSHLog-0-hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd-prefix:ba7f32532130,36361,1732491468451.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:14,799 WARN  [FSHLog-0-hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd-prefix:ba7f32532130,36361,1732491468451.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:14,800 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C36361%2C1732491468451.meta:.meta(num 1732491469669) roll requested
2024-11-24T23:38:14,800 INFO  [regionserver/ba7f32532130:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C36361%2C1732491468451.meta.1732491494800.meta
2024-11-24T23:38:14,805 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:14,805 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:14,805 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:14,806 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:14,806 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:14,806 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.meta.1732491469669.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.meta.1732491494800.meta
2024-11-24T23:38:14,807 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:14,807 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:14,807 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.meta.1732491469669.meta
2024-11-24T23:38:14,807 WARN  [IPC Server handler 4 on default port 45999 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.meta.1732491469669.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1014
2024-11-24T23:38:14,808 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.meta.1732491469669.meta after 1ms
2024-11-24T23:38:14,808 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41107:41107),(127.0.0.1/127.0.0.1:40201:40201)]
2024-11-24T23:38:14,808 DEBUG [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.meta.1732491469669.meta is not closed yet, will try archiving it next time
2024-11-24T23:38:14,823 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/.tmp/info/6b9717acee9e407bb0a062c1aeff0dea is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d./info:regioninfo/1732491470250/Put/seqid=0
2024-11-24T23:38:14,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741843_1027 (size=7125)
2024-11-24T23:38:14,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741843_1027 (size=7125)
2024-11-24T23:38:14,829 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/.tmp/info/6b9717acee9e407bb0a062c1aeff0dea
2024-11-24T23:38:14,846 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/.tmp/ns/380bb3ed9d024b7eb462ed74e599cc79 is 43, key is default/ns:d/1732491469840/Put/seqid=0
2024-11-24T23:38:14,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741844_1028 (size=5153)
2024-11-24T23:38:14,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741844_1028 (size=5153)
2024-11-24T23:38:14,851 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/.tmp/ns/380bb3ed9d024b7eb462ed74e599cc79
2024-11-24T23:38:14,871 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/.tmp/table/490514836b8943c0aa405c984a7f6e4d is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1732491470261/Put/seqid=0
2024-11-24T23:38:14,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741845_1029 (size=5438)
2024-11-24T23:38:14,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741845_1029 (size=5438)
2024-11-24T23:38:14,876 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/.tmp/table/490514836b8943c0aa405c984a7f6e4d
2024-11-24T23:38:14,881 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/.tmp/info/6b9717acee9e407bb0a062c1aeff0dea as hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/info/6b9717acee9e407bb0a062c1aeff0dea
2024-11-24T23:38:14,887 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/info/6b9717acee9e407bb0a062c1aeff0dea, entries=10, sequenceid=11, filesize=7.0 K
2024-11-24T23:38:14,888 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/.tmp/ns/380bb3ed9d024b7eb462ed74e599cc79 as hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/ns/380bb3ed9d024b7eb462ed74e599cc79
2024-11-24T23:38:14,894 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/ns/380bb3ed9d024b7eb462ed74e599cc79, entries=2, sequenceid=11, filesize=5.0 K
2024-11-24T23:38:14,895 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/.tmp/table/490514836b8943c0aa405c984a7f6e4d as hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/table/490514836b8943c0aa405c984a7f6e4d
2024-11-24T23:38:14,902 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/table/490514836b8943c0aa405c984a7f6e4d, entries=2, sequenceid=11, filesize=5.3 K
2024-11-24T23:38:14,903 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false
2024-11-24T23:38:14,903 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-24T23:38:14,908 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T23:38:14,908 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:38:14,908 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:38:14,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:38:14,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:38:14,908 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T23:38:14,908 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T23:38:14,909 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=33676800, stopped=false
2024-11-24T23:38:14,909 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ba7f32532130,43305,1732491468267
2024-11-24T23:38:14,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:38:14,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:38:14,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:14,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:14,971 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:38:14,971 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:38:14,972 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:38:14,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:38:14,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:38:14,973 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba7f32532130,36361,1732491468451' *****
2024-11-24T23:38:14,973 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T23:38:14,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:38:14,974 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T23:38:14,974 INFO  [RS:0;ba7f32532130:36361 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T23:38:14,974 INFO  [RS:0;ba7f32532130:36361 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T23:38:14,974 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T23:38:14,974 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(3091): Received CLOSE for dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:38:14,975 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(959): stopping server ba7f32532130,36361,1732491468451
2024-11-24T23:38:14,975 INFO  [RS:0;ba7f32532130:36361 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:38:14,975 INFO  [RS:0;ba7f32532130:36361 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ba7f32532130:36361.
2024-11-24T23:38:14,975 DEBUG [RS:0;ba7f32532130:36361 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:38:14,975 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dce5568d03590a86e4e8225b88370f8d, disabling compactions & flushes
2024-11-24T23:38:14,975 DEBUG [RS:0;ba7f32532130:36361 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:38:14,975 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:38:14,975 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:38:14,975 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T23:38:14,976 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T23:38:14,976 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d. after waiting 0 ms
2024-11-24T23:38:14,976 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T23:38:14,976 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:38:14,976 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T23:38:14,976 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-24T23:38:14,976 DEBUG [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(1325): Online Regions={dce5568d03590a86e4e8225b88370f8d=TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d., 1588230740=hbase:meta,,1.1588230740}
2024-11-24T23:38:14,976 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:38:14,977 DEBUG [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, dce5568d03590a86e4e8225b88370f8d
2024-11-24T23:38:14,977 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:38:14,977 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:38:14,977 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:38:14,977 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:38:14,982 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/default/TestLogRolling-testLogRollOnPipelineRestart/dce5568d03590a86e4e8225b88370f8d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-24T23:38:14,982 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-24T23:38:14,983 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:38:14,983 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dce5568d03590a86e4e8225b88370f8d:
Waiting for close lock at 1732491494975Running coprocessor pre-close hooks at 1732491494975Disabling compacts and flushes for region at 1732491494975Disabling writes for close at 1732491494976 (+1 ms)Writing region close event to WAL at 1732491494977 (+1 ms)Running coprocessor post-close hooks at 1732491494982 (+5 ms)Closed at 1732491494983 (+1 ms)
2024-11-24T23:38:14,983 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732491469889.dce5568d03590a86e4e8225b88370f8d.
2024-11-24T23:38:14,983 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:38:14,983 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:38:14,983 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491494976Running coprocessor pre-close hooks at 1732491494976Disabling compacts and flushes for region at 1732491494976Disabling writes for close at 1732491494977 (+1 ms)Writing region close event to WAL at 1732491494979 (+2 ms)Running coprocessor post-close hooks at 1732491494983 (+4 ms)Closed at 1732491494983
2024-11-24T23:38:14,983 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T23:38:15,177 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(976): stopping server ba7f32532130,36361,1732491468451; all regions closed.
2024-11-24T23:38:15,178 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:15,178 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:15,178 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:15,178 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:15,178 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:15,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741842_1025 (size=825)
2024-11-24T23:38:15,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741842_1025 (size=825)
2024-11-24T23:38:15,416 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:15,475 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:15,933 INFO  [regionserver/ba7f32532130:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-24T23:38:15,933 INFO  [regionserver/ba7f32532130:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-24T23:38:16,417 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:16,475 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:16,934 INFO  [regionserver/ba7f32532130:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:38:17,211 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-24T23:38:17,418 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:17,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:18,245 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T23:38:18,419 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:18,477 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:18,808 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.meta.1732491469669.meta after 4001ms
2024-11-24T23:38:18,809 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/WALs/ba7f32532130,36361,1732491468451/ba7f32532130%2C36361%2C1732491468451.meta.1732491469669.meta to hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/oldWALs/ba7f32532130%2C36361%2C1732491468451.meta.1732491469669.meta
2024-11-24T23:38:18,813 DEBUG [RS:0;ba7f32532130:36361 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/oldWALs
2024-11-24T23:38:18,813 INFO  [RS:0;ba7f32532130:36361 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C36361%2C1732491468451.meta:.meta(num 1732491494800)
2024-11-24T23:38:18,814 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:18,816 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:18,816 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:18,816 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:18,816 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:18,818 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741840_1023 (size=1162)
2024-11-24T23:38:18,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741840_1023 (size=1162)
2024-11-24T23:38:18,824 DEBUG [RS:0;ba7f32532130:36361 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/oldWALs
2024-11-24T23:38:18,824 INFO  [RS:0;ba7f32532130:36361 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C36361%2C1732491468451:(num 1732491494756)
2024-11-24T23:38:18,824 DEBUG [RS:0;ba7f32532130:36361 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:38:18,824 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:38:18,825 INFO  [RS:0;ba7f32532130:36361 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:38:18,825 INFO  [RS:0;ba7f32532130:36361 {}] hbase.ChoreService(370): Chore service for: regionserver/ba7f32532130:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T23:38:18,825 INFO  [RS:0;ba7f32532130:36361 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:38:18,825 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:38:18,825 INFO  [RS:0;ba7f32532130:36361 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36361
2024-11-24T23:38:18,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:38:18,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba7f32532130,36361,1732491468451
2024-11-24T23:38:18,844 INFO  [RS:0;ba7f32532130:36361 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:38:18,854 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba7f32532130,36361,1732491468451]
2024-11-24T23:38:18,864 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba7f32532130,36361,1732491468451 already deleted, retry=false
2024-11-24T23:38:18,865 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba7f32532130,36361,1732491468451 expired; onlineServers=0
2024-11-24T23:38:18,865 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ba7f32532130,43305,1732491468267' *****
2024-11-24T23:38:18,865 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T23:38:18,865 INFO  [M:0;ba7f32532130:43305 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:38:18,865 INFO  [M:0;ba7f32532130:43305 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:38:18,865 DEBUG [M:0;ba7f32532130:43305 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T23:38:18,865 DEBUG [M:0;ba7f32532130:43305 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T23:38:18,865 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T23:38:18,865 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491468800 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491468800,5,FailOnTimeoutGroup]
2024-11-24T23:38:18,865 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491468798 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491468798,5,FailOnTimeoutGroup]
2024-11-24T23:38:18,865 INFO  [M:0;ba7f32532130:43305 {}] hbase.ChoreService(370): Chore service for: master/ba7f32532130:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T23:38:18,866 INFO  [M:0;ba7f32532130:43305 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:38:18,866 DEBUG [M:0;ba7f32532130:43305 {}] master.HMaster(1795): Stopping service threads
2024-11-24T23:38:18,866 INFO  [M:0;ba7f32532130:43305 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T23:38:18,866 INFO  [M:0;ba7f32532130:43305 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:38:18,866 INFO  [M:0;ba7f32532130:43305 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T23:38:18,866 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T23:38:18,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T23:38:18,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:18,875 DEBUG [M:0;ba7f32532130:43305 {}] zookeeper.ZKUtil(347): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T23:38:18,875 WARN  [M:0;ba7f32532130:43305 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T23:38:18,876 INFO  [M:0;ba7f32532130:43305 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/.lastflushedseqids
2024-11-24T23:38:18,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741846_1030 (size=130)
2024-11-24T23:38:18,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741846_1030 (size=130)
2024-11-24T23:38:18,882 INFO  [M:0;ba7f32532130:43305 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T23:38:18,882 INFO  [M:0;ba7f32532130:43305 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T23:38:18,882 DEBUG [M:0;ba7f32532130:43305 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:38:18,882 INFO  [M:0;ba7f32532130:43305 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:38:18,882 DEBUG [M:0;ba7f32532130:43305 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:38:18,882 DEBUG [M:0;ba7f32532130:43305 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:38:18,882 DEBUG [M:0;ba7f32532130:43305 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:38:18,882 INFO  [M:0;ba7f32532130:43305 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.18 KB heapSize=29.16 KB
2024-11-24T23:38:18,883 ERROR [FSHLog-0-hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData-prefix:ba7f32532130,43305,1732491468267 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:18,883 WARN  [FSHLog-0-hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData-prefix:ba7f32532130,43305,1732491468267 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:18,883 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog ba7f32532130%2C43305%2C1732491468267:(num 1732491468572) roll requested
2024-11-24T23:38:18,883 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C43305%2C1732491468267.1732491498883
2024-11-24T23:38:18,888 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:18,888 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:18,889 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:18,889 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:18,889 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:18,889 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267/ba7f32532130%2C43305%2C1732491468267.1732491468572 with entries=53, filesize=26.63 KB; new WAL /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267/ba7f32532130%2C43305%2C1732491468267.1732491498883
2024-11-24T23:38:18,889 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:18,889 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46187,DS-48525ee5-9e2c-4af5-bcf8-760f18c95a13,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T23:38:18,890 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267/ba7f32532130%2C43305%2C1732491468267.1732491468572
2024-11-24T23:38:18,890 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41107:41107),(127.0.0.1/127.0.0.1:40201:40201)]
2024-11-24T23:38:18,890 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267/ba7f32532130%2C43305%2C1732491468267.1732491468572 is not closed yet, will try archiving it next time
2024-11-24T23:38:18,890 WARN  [IPC Server handler 2 on default port 45999 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267/ba7f32532130%2C43305%2C1732491468267.1732491468572 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-24T23:38:18,890 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267/ba7f32532130%2C43305%2C1732491468267.1732491468572 after 0ms
2024-11-24T23:38:18,903 DEBUG [M:0;ba7f32532130:43305 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1b790ec34e74c3faa0da657986e46a7 is 82, key is hbase:meta,,1/info:regioninfo/1732491469694/Put/seqid=0
2024-11-24T23:38:18,908 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741848_1033 (size=5672)
2024-11-24T23:38:18,908 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741848_1033 (size=5672)
2024-11-24T23:38:18,909 INFO  [M:0;ba7f32532130:43305 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1b790ec34e74c3faa0da657986e46a7
2024-11-24T23:38:18,927 DEBUG [M:0;ba7f32532130:43305 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b64f049c23344eaa68fbdb1cadb1531 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732491470267/Put/seqid=0
2024-11-24T23:38:18,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741849_1034 (size=6119)
2024-11-24T23:38:18,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741849_1034 (size=6119)
2024-11-24T23:38:18,932 INFO  [M:0;ba7f32532130:43305 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b64f049c23344eaa68fbdb1cadb1531
2024-11-24T23:38:18,949 DEBUG [M:0;ba7f32532130:43305 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa901a5614fc4fc28ade8b2f6baf0875 is 69, key is ba7f32532130,36361,1732491468451/rs:state/1732491468906/Put/seqid=0
2024-11-24T23:38:18,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741850_1035 (size=5156)
2024-11-24T23:38:18,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:38:18,954 INFO  [RS:0;ba7f32532130:36361 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:38:18,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x1016f35d9820001, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:38:18,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741850_1035 (size=5156)
2024-11-24T23:38:18,954 INFO  [RS:0;ba7f32532130:36361 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba7f32532130,36361,1732491468451; zookeeper connection closed.
2024-11-24T23:38:18,954 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@615de59f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@615de59f
2024-11-24T23:38:18,955 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T23:38:18,955 INFO  [M:0;ba7f32532130:43305 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa901a5614fc4fc28ade8b2f6baf0875
2024-11-24T23:38:18,973 DEBUG [M:0;ba7f32532130:43305 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/59d389eb9832484a8ca32fd68b9ee39c is 52, key is load_balancer_on/state:d/1732491469885/Put/seqid=0
2024-11-24T23:38:18,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741851_1036 (size=5056)
2024-11-24T23:38:18,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741851_1036 (size=5056)
2024-11-24T23:38:18,979 INFO  [M:0;ba7f32532130:43305 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/59d389eb9832484a8ca32fd68b9ee39c
2024-11-24T23:38:18,984 DEBUG [M:0;ba7f32532130:43305 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1b790ec34e74c3faa0da657986e46a7 as hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1b790ec34e74c3faa0da657986e46a7
2024-11-24T23:38:18,990 INFO  [M:0;ba7f32532130:43305 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1b790ec34e74c3faa0da657986e46a7, entries=8, sequenceid=56, filesize=5.5 K
2024-11-24T23:38:18,990 DEBUG [M:0;ba7f32532130:43305 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b64f049c23344eaa68fbdb1cadb1531 as hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5b64f049c23344eaa68fbdb1cadb1531
2024-11-24T23:38:18,996 INFO  [M:0;ba7f32532130:43305 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5b64f049c23344eaa68fbdb1cadb1531, entries=6, sequenceid=56, filesize=6.0 K
2024-11-24T23:38:18,997 DEBUG [M:0;ba7f32532130:43305 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa901a5614fc4fc28ade8b2f6baf0875 as hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fa901a5614fc4fc28ade8b2f6baf0875
2024-11-24T23:38:19,002 INFO  [M:0;ba7f32532130:43305 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fa901a5614fc4fc28ade8b2f6baf0875, entries=1, sequenceid=56, filesize=5.0 K
2024-11-24T23:38:19,003 DEBUG [M:0;ba7f32532130:43305 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/59d389eb9832484a8ca32fd68b9ee39c as hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/59d389eb9832484a8ca32fd68b9ee39c
2024-11-24T23:38:19,008 INFO  [M:0;ba7f32532130:43305 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/59d389eb9832484a8ca32fd68b9ee39c, entries=1, sequenceid=56, filesize=4.9 K
2024-11-24T23:38:19,009 INFO  [M:0;ba7f32532130:43305 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 127ms, sequenceid=56, compaction requested=false
2024-11-24T23:38:19,011 INFO  [M:0;ba7f32532130:43305 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:38:19,011 DEBUG [M:0;ba7f32532130:43305 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491498882Disabling compacts and flushes for region at 1732491498882Disabling writes for close at 1732491498882Obtaining lock to block concurrent updates at 1732491498882Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732491498882Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23738, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1732491498883 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732491498890 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732491498890Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732491498903 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732491498903Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732491498913 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732491498927 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732491498927Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732491498936 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732491498949 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732491498949Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732491498959 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732491498972 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732491498972Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51cf5472: reopening flushed file at 1732491498984 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26406cd3: reopening flushed file at 1732491498990 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c276f: reopening flushed file at 1732491498996 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19904f08: reopening flushed file at 1732491499002 (+6 ms)Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 127ms, sequenceid=56, compaction requested=false at 1732491499009 (+7 ms)Writing region close event to WAL at 1732491499011 (+2 ms)Closed at 1732491499011
2024-11-24T23:38:19,011 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:19,011 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:19,011 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:19,011 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:19,011 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:19,013 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42057 is added to blk_1073741847_1031 (size=757)
2024-11-24T23:38:19,013 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38923 is added to blk_1073741847_1031 (size=757)
2024-11-24T23:38:19,420 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:19,479 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:19,984 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:19,985 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,006 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,006 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,006 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,006 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,006 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,007 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,010 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,011 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,011 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,014 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,022 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,022 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,212 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-24T23:38:20,420 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:20,479 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:20,525 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T23:38:20,526 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,526 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,527 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,527 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,542 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,543 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,543 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,543 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,543 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,544 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,547 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,547 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,547 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:20,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:21,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:21,480 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:22,422 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:22,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:22,891 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267/ba7f32532130%2C43305%2C1732491468267.1732491468572 after 4001ms
2024-11-24T23:38:22,892 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/WALs/ba7f32532130,43305,1732491468267/ba7f32532130%2C43305%2C1732491468267.1732491468572 to hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/oldWALs/ba7f32532130%2C43305%2C1732491468267.1732491468572
2024-11-24T23:38:22,895 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/MasterData/oldWALs/ba7f32532130%2C43305%2C1732491468267.1732491468572 to hdfs://localhost:45999/user/jenkins/test-data/3dfbcdc6-31ba-4977-8bd0-8273fb5de3dd/oldWALs/ba7f32532130%2C43305%2C1732491468267.1732491468572$masterlocalwal$
2024-11-24T23:38:22,895 INFO  [M:0;ba7f32532130:43305 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T23:38:22,895 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:38:22,895 INFO  [M:0;ba7f32532130:43305 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43305
2024-11-24T23:38:22,896 INFO  [M:0;ba7f32532130:43305 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:38:23,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:38:23,007 INFO  [M:0;ba7f32532130:43305 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:38:23,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43305-0x1016f35d9820000, quorum=127.0.0.1:61696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:38:23,009 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f03ce36{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:23,009 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@cbccde7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:38:23,010 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:38:23,010 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3df3f65e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:38:23,010 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bc7279c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,STOPPED}
2024-11-24T23:38:23,011 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:38:23,011 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:38:23,011 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556390239-172.17.0.3-1732491465666 (Datanode Uuid 73182778-e1ce-4aaa-9ee7-4b3063186b65) service to localhost/127.0.0.1:45999
2024-11-24T23:38:23,011 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:38:23,011 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data3/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:23,012 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data4/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:23,012 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:38:23,014 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@77a301c7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:23,014 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7c0d41d9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:38:23,014 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:38:23,015 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22abc971{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:38:23,015 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@518d8d55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,STOPPED}
2024-11-24T23:38:23,016 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:38:23,016 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:38:23,016 WARN  [BP-1556390239-172.17.0.3-1732491465666 heartbeating to localhost/127.0.0.1:45999 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556390239-172.17.0.3-1732491465666 (Datanode Uuid 503b6869-be40-49d2-984d-e9a7c62761c2) service to localhost/127.0.0.1:45999
2024-11-24T23:38:23,016 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:38:23,017 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data1/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:23,017 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/cluster_103c05ae-221c-d005-be7c-af6039d1f78d/data/data2/current/BP-1556390239-172.17.0.3-1732491465666 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:38:23,017 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:38:23,024 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f78fbde{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:38:23,024 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@53effb5a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:38:23,024 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:38:23,024 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e4cbcc2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:38:23,024 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7e13d66c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir/,STOPPED}
2024-11-24T23:38:23,029 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T23:38:23,046 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T23:38:23,054 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=182 (was 158)
Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:45999
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45999 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45999
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45999
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45999
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:45999
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45999 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45999 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=455 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=298 (was 268) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3310 (was 4600)
2024-11-24T23:38:23,061 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=182, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=298, ProcessCount=11, AvailableMemoryMB=3310
2024-11-24T23:38:23,061 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T23:38:23,062 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.log.dir so I do NOT create it in target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32
2024-11-24T23:38:23,062 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/569d577a-f5d7-f7ab-5928-9aa5b486a261/hadoop.tmp.dir so I do NOT create it in target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32
2024-11-24T23:38:23,062 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f, deleteOnExit=true
2024-11-24T23:38:23,062 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T23:38:23,062 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/test.cache.data in system properties and HBase conf
2024-11-24T23:38:23,062 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T23:38:23,062 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/hadoop.log.dir in system properties and HBase conf
2024-11-24T23:38:23,062 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T23:38:23,062 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T23:38:23,062 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T23:38:23,063 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T23:38:23,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:38:23,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:38:23,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T23:38:23,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:38:23,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T23:38:23,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T23:38:23,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:38:23,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:38:23,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T23:38:23,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/nfs.dump.dir in system properties and HBase conf
2024-11-24T23:38:23,064 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/java.io.tmpdir in system properties and HBase conf
2024-11-24T23:38:23,064 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:38:23,064 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T23:38:23,064 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T23:38:23,078 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:38:23,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:38:23,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T23:38:23,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-24T23:38:23,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-24T23:38:23,423 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:23,452 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:38:23,459 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:38:23,460 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:38:23,460 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:38:23,460 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:38:23,461 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:38:23,464 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a4b134d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:38:23,465 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc3ea71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:38:23,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:23,571 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2719a663{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/java.io.tmpdir/jetty-localhost-35603-hadoop-hdfs-3_4_1-tests_jar-_-any-1557989578920959950/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:38:23,572 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1af676f5{HTTP/1.1, (http/1.1)}{localhost:35603}
2024-11-24T23:38:23,572 INFO  [Time-limited test {}] server.Server(415): Started @198156ms
2024-11-24T23:38:23,586 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:38:23,907 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:38:23,911 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:38:23,913 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:38:23,913 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:38:23,913 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:38:23,914 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39f00f78{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:38:23,914 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74b5ebca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:38:24,022 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5127cbf0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/java.io.tmpdir/jetty-localhost-41491-hadoop-hdfs-3_4_1-tests_jar-_-any-11438547952499322171/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:24,023 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@9b25e94{HTTP/1.1, (http/1.1)}{localhost:41491}
2024-11-24T23:38:24,023 INFO  [Time-limited test {}] server.Server(415): Started @198606ms
2024-11-24T23:38:24,024 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:38:24,078 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:38:24,081 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:38:24,086 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:38:24,086 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:38:24,086 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:38:24,087 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ec76923{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:38:24,087 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f411ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:38:24,202 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c6abbb8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/java.io.tmpdir/jetty-localhost-41069-hadoop-hdfs-3_4_1-tests_jar-_-any-4881894881136925790/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:38:24,203 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@555a4a92{HTTP/1.1, (http/1.1)}{localhost:41069}
2024-11-24T23:38:24,203 INFO  [Time-limited test {}] server.Server(415): Started @198787ms
2024-11-24T23:38:24,204 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:38:24,423 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:24,483 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:25,127 WARN  [Thread-1644 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f/data/data2/current/BP-324770769-172.17.0.3-1732491503088/current, will proceed with Du for space computation calculation, 
2024-11-24T23:38:25,127 WARN  [Thread-1643 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f/data/data1/current/BP-324770769-172.17.0.3-1732491503088/current, will proceed with Du for space computation calculation, 
2024-11-24T23:38:25,156 WARN  [Thread-1607 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:38:25,159 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe0733e99cafe8355 with lease ID 0xb580e10259ffed74: Processing first storage report for DS-eb4cb439-a001-48d7-81ed-40d8e17f101b from datanode DatanodeRegistration(127.0.0.1:40221, datanodeUuid=4bcc870c-3aa6-4c01-b5e1-759d40dfd57e, infoPort=33113, infoSecurePort=0, ipcPort=38689, storageInfo=lv=-57;cid=testClusterID;nsid=1424706991;c=1732491503088)
2024-11-24T23:38:25,159 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe0733e99cafe8355 with lease ID 0xb580e10259ffed74: from storage DS-eb4cb439-a001-48d7-81ed-40d8e17f101b node DatanodeRegistration(127.0.0.1:40221, datanodeUuid=4bcc870c-3aa6-4c01-b5e1-759d40dfd57e, infoPort=33113, infoSecurePort=0, ipcPort=38689, storageInfo=lv=-57;cid=testClusterID;nsid=1424706991;c=1732491503088), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:38:25,159 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe0733e99cafe8355 with lease ID 0xb580e10259ffed74: Processing first storage report for DS-096589d5-948c-4c37-91e2-4c8356c53823 from datanode DatanodeRegistration(127.0.0.1:40221, datanodeUuid=4bcc870c-3aa6-4c01-b5e1-759d40dfd57e, infoPort=33113, infoSecurePort=0, ipcPort=38689, storageInfo=lv=-57;cid=testClusterID;nsid=1424706991;c=1732491503088)
2024-11-24T23:38:25,159 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe0733e99cafe8355 with lease ID 0xb580e10259ffed74: from storage DS-096589d5-948c-4c37-91e2-4c8356c53823 node DatanodeRegistration(127.0.0.1:40221, datanodeUuid=4bcc870c-3aa6-4c01-b5e1-759d40dfd57e, infoPort=33113, infoSecurePort=0, ipcPort=38689, storageInfo=lv=-57;cid=testClusterID;nsid=1424706991;c=1732491503088), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:38:25,319 WARN  [Thread-1654 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f/data/data3/current/BP-324770769-172.17.0.3-1732491503088/current, will proceed with Du for space computation calculation, 
2024-11-24T23:38:25,319 WARN  [Thread-1655 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f/data/data4/current/BP-324770769-172.17.0.3-1732491503088/current, will proceed with Du for space computation calculation, 
2024-11-24T23:38:25,335 WARN  [Thread-1630 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:38:25,337 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84d9ea7b20db75a4 with lease ID 0xb580e10259ffed75: Processing first storage report for DS-4231e069-af4b-4986-831d-a05ff5db59a3 from datanode DatanodeRegistration(127.0.0.1:44333, datanodeUuid=81a40776-8776-4730-953a-9a71b59ec274, infoPort=45075, infoSecurePort=0, ipcPort=33153, storageInfo=lv=-57;cid=testClusterID;nsid=1424706991;c=1732491503088)
2024-11-24T23:38:25,337 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84d9ea7b20db75a4 with lease ID 0xb580e10259ffed75: from storage DS-4231e069-af4b-4986-831d-a05ff5db59a3 node DatanodeRegistration(127.0.0.1:44333, datanodeUuid=81a40776-8776-4730-953a-9a71b59ec274, infoPort=45075, infoSecurePort=0, ipcPort=33153, storageInfo=lv=-57;cid=testClusterID;nsid=1424706991;c=1732491503088), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:38:25,337 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84d9ea7b20db75a4 with lease ID 0xb580e10259ffed75: Processing first storage report for DS-b818ff7f-e3df-4c78-ac51-dd5769907941 from datanode DatanodeRegistration(127.0.0.1:44333, datanodeUuid=81a40776-8776-4730-953a-9a71b59ec274, infoPort=45075, infoSecurePort=0, ipcPort=33153, storageInfo=lv=-57;cid=testClusterID;nsid=1424706991;c=1732491503088)
2024-11-24T23:38:25,337 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84d9ea7b20db75a4 with lease ID 0xb580e10259ffed75: from storage DS-b818ff7f-e3df-4c78-ac51-dd5769907941 node DatanodeRegistration(127.0.0.1:44333, datanodeUuid=81a40776-8776-4730-953a-9a71b59ec274, infoPort=45075, infoSecurePort=0, ipcPort=33153, storageInfo=lv=-57;cid=testClusterID;nsid=1424706991;c=1732491503088), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:38:25,344 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32
2024-11-24T23:38:25,353 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f/zookeeper_0, clientPort=53988, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T23:38:25,354 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53988
2024-11-24T23:38:25,354 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:38:25,355 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:38:25,370 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:38:25,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:38:25,372 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83 with version=8
2024-11-24T23:38:25,372 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/hbase-staging
2024-11-24T23:38:25,374 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:38:25,375 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:38:25,375 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:38:25,375 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:38:25,375 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:38:25,375 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:38:25,375 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T23:38:25,375 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:38:25,376 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35219
2024-11-24T23:38:25,379 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35219 connecting to ZooKeeper ensemble=127.0.0.1:53988
2024-11-24T23:38:25,424 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:25,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:352190x0, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:38:25,435 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35219-0x1016f366a6e0000 connected
2024-11-24T23:38:25,483 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:25,770 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:38:25,772 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:38:25,773 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:38:25,774 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83, hbase.cluster.distributed=false
2024-11-24T23:38:25,775 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:38:25,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35219
2024-11-24T23:38:25,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35219
2024-11-24T23:38:25,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35219
2024-11-24T23:38:25,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35219
2024-11-24T23:38:25,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35219
2024-11-24T23:38:25,794 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:38:25,794 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:38:25,794 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:38:25,794 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:38:25,794 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:38:25,794 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:38:25,794 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T23:38:25,794 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:38:25,795 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40747
2024-11-24T23:38:25,797 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40747 connecting to ZooKeeper ensemble=127.0.0.1:53988
2024-11-24T23:38:25,798 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:38:25,799 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:38:25,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407470x0, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:38:25,813 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:407470x0, quorum=127.0.0.1:53988, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:38:25,813 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40747-0x1016f366a6e0001 connected
2024-11-24T23:38:25,813 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T23:38:25,814 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T23:38:25,814 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T23:38:25,815 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:38:25,815 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40747
2024-11-24T23:38:25,815 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40747
2024-11-24T23:38:25,815 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40747
2024-11-24T23:38:25,816 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40747
2024-11-24T23:38:25,816 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40747
2024-11-24T23:38:25,828 DEBUG [M:0;ba7f32532130:35219 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ba7f32532130:35219
2024-11-24T23:38:25,828 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ba7f32532130,35219,1732491505374
2024-11-24T23:38:25,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:38:25,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:38:25,834 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ba7f32532130,35219,1732491505374
2024-11-24T23:38:25,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T23:38:25,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:25,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:25,844 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T23:38:25,845 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ba7f32532130,35219,1732491505374 from backup master directory
2024-11-24T23:38:25,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ba7f32532130,35219,1732491505374
2024-11-24T23:38:25,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:38:25,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:38:25,854 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:38:25,854 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ba7f32532130,35219,1732491505374
2024-11-24T23:38:25,858 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/hbase.id] with ID: 65043a79-68f7-477f-adcb-74f93f9b3d6a
2024-11-24T23:38:25,858 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/.tmp/hbase.id
2024-11-24T23:38:25,862 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:38:25,863 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:38:25,863 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/.tmp/hbase.id]:[hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/hbase.id]
2024-11-24T23:38:25,873 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:38:25,873 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T23:38:25,874 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-24T23:38:25,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:25,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:25,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:38:25,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:38:25,931 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:38:25,931 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T23:38:25,932 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:38:25,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:38:25,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:38:25,943 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store
2024-11-24T23:38:25,950 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:38:25,950 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:38:25,950 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:38:25,950 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:38:25,951 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:38:25,951 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:38:25,951 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:38:25,951 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:38:25,951 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:38:25,951 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491505950Disabling compacts and flushes for region at 1732491505950Disabling writes for close at 1732491505951 (+1 ms)Writing region close event to WAL at 1732491505951Closed at 1732491505951
2024-11-24T23:38:25,951 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/.initializing
2024-11-24T23:38:25,952 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/WALs/ba7f32532130,35219,1732491505374
2024-11-24T23:38:25,954 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C35219%2C1732491505374, suffix=, logDir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/WALs/ba7f32532130,35219,1732491505374, archiveDir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/oldWALs, maxLogs=10
2024-11-24T23:38:25,954 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C35219%2C1732491505374.1732491505954
2024-11-24T23:38:25,959 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/WALs/ba7f32532130,35219,1732491505374/ba7f32532130%2C35219%2C1732491505374.1732491505954
2024-11-24T23:38:25,960 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33113:33113),(127.0.0.1/127.0.0.1:45075:45075)]
2024-11-24T23:38:25,963 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:38:25,964 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:38:25,964 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:38:25,964 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:38:25,965 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:38:25,967 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T23:38:25,967 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:25,967 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:38:25,967 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:38:25,969 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T23:38:25,969 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:25,969 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:38:25,969 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:38:25,971 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T23:38:25,971 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:25,972 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:38:25,972 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:38:25,973 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T23:38:25,973 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:25,974 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:38:25,974 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:38:25,975 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:38:25,975 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:38:25,976 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:38:25,976 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:38:25,977 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T23:38:25,978 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:38:25,980 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:38:25,980 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=772513, jitterRate=-0.017699241638183594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T23:38:25,981 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732491505964Initializing all the Stores at 1732491505965 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491505965Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491505965Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491505965Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491505965Cleaning up temporary data from old regions at 1732491505976 (+11 ms)Region opened successfully at 1732491505981 (+5 ms)
2024-11-24T23:38:25,981 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T23:38:25,988 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542a762c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:38:25,989 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T23:38:25,989 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T23:38:25,989 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T23:38:25,990 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T23:38:25,990 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T23:38:25,991 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T23:38:25,991 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T23:38:25,995 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T23:38:25,996 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T23:38:26,033 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T23:38:26,034 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T23:38:26,034 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T23:38:26,044 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T23:38:26,044 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T23:38:26,045 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T23:38:26,054 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T23:38:26,055 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T23:38:26,065 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T23:38:26,067 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T23:38:26,075 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T23:38:26,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:38:26,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:38:26,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:26,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:26,086 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ba7f32532130,35219,1732491505374, sessionid=0x1016f366a6e0000, setting cluster-up flag (Was=false)
2024-11-24T23:38:26,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:26,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:26,139 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T23:38:26,140 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,35219,1732491505374
2024-11-24T23:38:26,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:26,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:26,191 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T23:38:26,192 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,35219,1732491505374
2024-11-24T23:38:26,193 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T23:38:26,195 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T23:38:26,195 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T23:38:26,195 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T23:38:26,195 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ba7f32532130,35219,1732491505374
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T23:38:26,197 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:38:26,197 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:38:26,197 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:38:26,197 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:38:26,197 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ba7f32532130:0, corePoolSize=10, maxPoolSize=10
2024-11-24T23:38:26,197 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,197 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:38:26,197 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,201 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732491536201
2024-11-24T23:38:26,201 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T23:38:26,202 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T23:38:26,202 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T23:38:26,202 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T23:38:26,202 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T23:38:26,202 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T23:38:26,202 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:38:26,202 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,202 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T23:38:26,203 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:26,203 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T23:38:26,204 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T23:38:26,204 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T23:38:26,205 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T23:38:26,207 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T23:38:26,207 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T23:38:26,216 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491506207,5,FailOnTimeoutGroup]
2024-11-24T23:38:26,222 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491506216,5,FailOnTimeoutGroup]
2024-11-24T23:38:26,222 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,222 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T23:38:26,222 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,222 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,223 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(746): ClusterId : 65043a79-68f7-477f-adcb-74f93f9b3d6a
2024-11-24T23:38:26,223 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T23:38:26,234 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T23:38:26,234 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T23:38:26,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:38:26,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:38:26,239 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T23:38:26,240 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83
2024-11-24T23:38:26,245 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T23:38:26,245 DEBUG [RS:0;ba7f32532130:40747 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f3e917, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:38:26,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:38:26,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:38:26,257 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:38:26,263 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:38:26,265 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:38:26,265 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:26,266 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:38:26,266 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:38:26,266 DEBUG [RS:0;ba7f32532130:40747 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ba7f32532130:40747
2024-11-24T23:38:26,266 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T23:38:26,266 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T23:38:26,266 DEBUG [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T23:38:26,267 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba7f32532130,35219,1732491505374 with port=40747, startcode=1732491505794
2024-11-24T23:38:26,267 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:38:26,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:26,267 DEBUG [RS:0;ba7f32532130:40747 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T23:38:26,268 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:38:26,268 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:38:26,270 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:38:26,270 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:26,270 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:38:26,270 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:38:26,272 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:38:26,272 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:26,272 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:38:26,273 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:38:26,273 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55583, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T23:38:26,274 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35219 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba7f32532130,40747,1732491505794
2024-11-24T23:38:26,274 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35219 {}] master.ServerManager(517): Registering regionserver=ba7f32532130,40747,1732491505794
2024-11-24T23:38:26,275 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740
2024-11-24T23:38:26,275 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740
2024-11-24T23:38:26,276 DEBUG [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83
2024-11-24T23:38:26,276 DEBUG [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33195
2024-11-24T23:38:26,276 DEBUG [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T23:38:26,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:38:26,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:38:26,278 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:38:26,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:38:26,283 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:38:26,283 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715666, jitterRate=-0.08998483419418335}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:38:26,284 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732491506257Initializing all the Stores at 1732491506258 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491506258Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491506263 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491506263Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491506263Cleaning up temporary data from old regions at 1732491506277 (+14 ms)Region opened successfully at 1732491506284 (+7 ms)
2024-11-24T23:38:26,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:38:26,285 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:38:26,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:38:26,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:38:26,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:38:26,285 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:38:26,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491506284Disabling compacts and flushes for region at 1732491506284Disabling writes for close at 1732491506285 (+1 ms)Writing region close event to WAL at 1732491506285Closed at 1732491506285
2024-11-24T23:38:26,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:38:26,286 DEBUG [RS:0;ba7f32532130:40747 {}] zookeeper.ZKUtil(111): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba7f32532130,40747,1732491505794
2024-11-24T23:38:26,287 WARN  [RS:0;ba7f32532130:40747 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:38:26,287 INFO  [RS:0;ba7f32532130:40747 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:38:26,287 DEBUG [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794
2024-11-24T23:38:26,287 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba7f32532130,40747,1732491505794]
2024-11-24T23:38:26,288 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:38:26,288 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T23:38:26,288 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T23:38:26,290 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:38:26,291 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T23:38:26,299 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T23:38:26,303 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T23:38:26,304 INFO  [RS:0;ba7f32532130:40747 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T23:38:26,304 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,304 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T23:38:26,305 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T23:38:26,305 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,305 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,305 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,305 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,305 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,305 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,305 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:38:26,306 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,306 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,306 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,306 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,306 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,306 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:38:26,306 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:38:26,306 DEBUG [RS:0;ba7f32532130:40747 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:38:26,312 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,312 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,312 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,312 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,312 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,312 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,40747,1732491505794-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:38:26,330 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T23:38:26,331 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,40747,1732491505794-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,331 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,331 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.Replication(171): ba7f32532130,40747,1732491505794 started
2024-11-24T23:38:26,347 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:26,347 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(1482): Serving as ba7f32532130,40747,1732491505794, RpcServer on ba7f32532130/172.17.0.3:40747, sessionid=0x1016f366a6e0001
2024-11-24T23:38:26,347 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T23:38:26,347 DEBUG [RS:0;ba7f32532130:40747 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba7f32532130,40747,1732491505794
2024-11-24T23:38:26,347 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,40747,1732491505794'
2024-11-24T23:38:26,347 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T23:38:26,348 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T23:38:26,348 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T23:38:26,348 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T23:38:26,349 DEBUG [RS:0;ba7f32532130:40747 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba7f32532130,40747,1732491505794
2024-11-24T23:38:26,349 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,40747,1732491505794'
2024-11-24T23:38:26,349 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T23:38:26,349 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T23:38:26,349 DEBUG [RS:0;ba7f32532130:40747 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T23:38:26,350 INFO  [RS:0;ba7f32532130:40747 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T23:38:26,350 INFO  [RS:0;ba7f32532130:40747 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T23:38:26,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:26,441 WARN  [ba7f32532130:35219 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T23:38:26,452 INFO  [RS:0;ba7f32532130:40747 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C40747%2C1732491505794, suffix=, logDir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794, archiveDir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/oldWALs, maxLogs=32
2024-11-24T23:38:26,452 INFO  [RS:0;ba7f32532130:40747 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C40747%2C1732491505794.1732491506452
2024-11-24T23:38:26,458 INFO  [RS:0;ba7f32532130:40747 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491506452
2024-11-24T23:38:26,459 DEBUG [RS:0;ba7f32532130:40747 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45075:45075),(127.0.0.1/127.0.0.1:33113:33113)]
2024-11-24T23:38:26,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:26,692 DEBUG [ba7f32532130:35219 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T23:38:26,692 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba7f32532130,40747,1732491505794
2024-11-24T23:38:26,694 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,40747,1732491505794, state=OPENING
2024-11-24T23:38:26,796 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T23:38:26,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:26,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:38:26,876 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:38:26,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:38:26,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:38:26,876 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,40747,1732491505794}]
2024-11-24T23:38:27,032 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T23:38:27,036 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60297, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T23:38:27,039 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T23:38:27,039 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:38:27,041 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C40747%2C1732491505794.meta, suffix=.meta, logDir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794, archiveDir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/oldWALs, maxLogs=32
2024-11-24T23:38:27,041 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C40747%2C1732491505794.meta.1732491507041.meta
2024-11-24T23:38:27,046 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.meta.1732491507041.meta
2024-11-24T23:38:27,048 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45075:45075),(127.0.0.1/127.0.0.1:33113:33113)]
2024-11-24T23:38:27,050 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:38:27,050 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T23:38:27,050 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T23:38:27,050 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T23:38:27,051 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T23:38:27,051 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:38:27,051 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T23:38:27,051 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T23:38:27,052 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:38:27,053 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:38:27,053 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:27,054 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:38:27,054 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:38:27,054 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:38:27,054 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:27,055 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:38:27,055 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:38:27,055 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:38:27,055 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:27,056 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:38:27,056 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:38:27,056 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:38:27,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:27,057 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:38:27,057 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:38:27,057 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740
2024-11-24T23:38:27,058 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740
2024-11-24T23:38:27,060 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:38:27,060 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:38:27,060 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:38:27,061 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:38:27,062 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=694683, jitterRate=-0.11666585505008698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:38:27,062 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T23:38:27,063 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732491507051Writing region info on filesystem at 1732491507051Initializing all the Stores at 1732491507052 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491507052Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491507052Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491507052Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491507052Cleaning up temporary data from old regions at 1732491507060 (+8 ms)Running coprocessor post-open hooks at 1732491507062 (+2 ms)Region opened successfully at 1732491507063 (+1 ms)
2024-11-24T23:38:27,064 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732491507032
2024-11-24T23:38:27,066 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T23:38:27,067 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T23:38:27,067 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,40747,1732491505794
2024-11-24T23:38:27,068 INFO  [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,40747,1732491505794, state=OPEN
2024-11-24T23:38:27,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:38:27,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:38:27,126 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba7f32532130,40747,1732491505794
2024-11-24T23:38:27,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:38:27,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:38:27,130 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T23:38:27,130 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,40747,1732491505794 in 250 msec
2024-11-24T23:38:27,133 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T23:38:27,133 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 842 msec
2024-11-24T23:38:27,134 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:38:27,134 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T23:38:27,136 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:38:27,136 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,40747,1732491505794, seqNum=-1]
2024-11-24T23:38:27,136 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:38:27,137 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46289, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:38:27,144 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 948 msec
2024-11-24T23:38:27,144 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732491507144, completionTime=-1
2024-11-24T23:38:27,144 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T23:38:27,144 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T23:38:27,146 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T23:38:27,147 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732491567146
2024-11-24T23:38:27,147 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732491627147
2024-11-24T23:38:27,147 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-24T23:38:27,147 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35219,1732491505374-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:27,147 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35219,1732491505374-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:27,147 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35219,1732491505374-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:27,147 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ba7f32532130:35219, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:27,147 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:27,148 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:27,149 DEBUG [master/ba7f32532130:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T23:38:27,152 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.297sec
2024-11-24T23:38:27,152 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T23:38:27,152 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T23:38:27,152 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T23:38:27,152 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T23:38:27,152 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T23:38:27,152 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35219,1732491505374-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:38:27,152 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35219,1732491505374-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T23:38:27,154 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T23:38:27,154 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T23:38:27,154 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35219,1732491505374-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:38:27,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2494db76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:38:27,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ba7f32532130,35219,-1 for getting cluster id
2024-11-24T23:38:27,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T23:38:27,226 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65043a79-68f7-477f-adcb-74f93f9b3d6a'
2024-11-24T23:38:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T23:38:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65043a79-68f7-477f-adcb-74f93f9b3d6a"

2024-11-24T23:38:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27c61206, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:38:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ba7f32532130,35219,-1]
2024-11-24T23:38:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T23:38:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:38:27,229 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T23:38:27,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff9b03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:38:27,231 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:38:27,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,40747,1732491505794, seqNum=-1]
2024-11-24T23:38:27,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:38:27,233 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:38:27,235 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ba7f32532130,35219,1732491505374
2024-11-24T23:38:27,236 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:38:27,239 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T23:38:27,239 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-24T23:38:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is ba7f32532130,35219,1732491505374
2024-11-24T23:38:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5dd31aba
2024-11-24T23:38:27,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-24T23:38:27,242 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-24T23:38:27,243 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-24T23:38:27,243 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-24T23:38:27,243 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:38:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:38:27,246 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-24T23:38:27,246 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:27,246 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-24T23:38:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T23:38:27,250 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-24T23:38:27,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741835_1011 (size=405)
2024-11-24T23:38:27,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741835_1011 (size=405)
2024-11-24T23:38:27,264 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fcaa8b9d74a4b9f994bb576114fa7b0c, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83
2024-11-24T23:38:27,276 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741836_1012 (size=88)
2024-11-24T23:38:27,276 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741836_1012 (size=88)
2024-11-24T23:38:27,276 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:38:27,277 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing fcaa8b9d74a4b9f994bb576114fa7b0c, disabling compactions & flushes
2024-11-24T23:38:27,277 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:27,277 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:27,277 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c. after waiting 0 ms
2024-11-24T23:38:27,277 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:27,277 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:27,277 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for fcaa8b9d74a4b9f994bb576114fa7b0c:
Waiting for close lock at 1732491507277Disabling compacts and flushes for region at 1732491507277Disabling writes for close at 1732491507277Writing region close event to WAL at 1732491507277Closed at 1732491507277
2024-11-24T23:38:27,278 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-24T23:38:27,279 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1732491507278"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732491507278"}]},"ts":"1732491507278"}
2024-11-24T23:38:27,281 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-24T23:38:27,283 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-24T23:38:27,283 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732491507283"}]},"ts":"1732491507283"}
2024-11-24T23:38:27,285 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-24T23:38:27,286 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=fcaa8b9d74a4b9f994bb576114fa7b0c, ASSIGN}]
2024-11-24T23:38:27,287 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=fcaa8b9d74a4b9f994bb576114fa7b0c, ASSIGN
2024-11-24T23:38:27,288 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=fcaa8b9d74a4b9f994bb576114fa7b0c, ASSIGN; state=OFFLINE, location=ba7f32532130,40747,1732491505794; forceNewPlan=false, retain=false
2024-11-24T23:38:27,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:27,439 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fcaa8b9d74a4b9f994bb576114fa7b0c, regionState=OPENING, regionLocation=ba7f32532130,40747,1732491505794
2024-11-24T23:38:27,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=fcaa8b9d74a4b9f994bb576114fa7b0c, ASSIGN because future has completed
2024-11-24T23:38:27,442 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fcaa8b9d74a4b9f994bb576114fa7b0c, server=ba7f32532130,40747,1732491505794}]
2024-11-24T23:38:27,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:27,599 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:27,599 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fcaa8b9d74a4b9f994bb576114fa7b0c, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:38:27,600 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:38:27,600 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:38:27,600 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:38:27,600 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:38:27,602 INFO  [StoreOpener-fcaa8b9d74a4b9f994bb576114fa7b0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region fcaa8b9d74a4b9f994bb576114fa7b0c 
2024-11-24T23:38:27,603 INFO  [StoreOpener-fcaa8b9d74a4b9f994bb576114fa7b0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fcaa8b9d74a4b9f994bb576114fa7b0c columnFamilyName info
2024-11-24T23:38:27,603 DEBUG [StoreOpener-fcaa8b9d74a4b9f994bb576114fa7b0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:38:27,603 INFO  [StoreOpener-fcaa8b9d74a4b9f994bb576114fa7b0c-1 {}] regionserver.HStore(327): Store=fcaa8b9d74a4b9f994bb576114fa7b0c/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:38:27,604 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:38:27,604 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:38:27,605 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:38:27,605 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:38:27,605 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:38:27,607 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:38:27,609 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:38:27,609 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fcaa8b9d74a4b9f994bb576114fa7b0c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=877197, jitterRate=0.11541472375392914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T23:38:27,610 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:38:27,610 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fcaa8b9d74a4b9f994bb576114fa7b0c:
Running coprocessor pre-open hook at 1732491507600Writing region info on filesystem at 1732491507600Initializing all the Stores at 1732491507601 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491507601Cleaning up temporary data from old regions at 1732491507605 (+4 ms)Running coprocessor post-open hooks at 1732491507610 (+5 ms)Region opened successfully at 1732491507610
2024-11-24T23:38:27,612 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c., pid=6, masterSystemTime=1732491507595
2024-11-24T23:38:27,615 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:27,615 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:27,616 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fcaa8b9d74a4b9f994bb576114fa7b0c, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,40747,1732491505794
2024-11-24T23:38:27,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fcaa8b9d74a4b9f994bb576114fa7b0c, server=ba7f32532130,40747,1732491505794 because future has completed
2024-11-24T23:38:27,629 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-24T23:38:27,629 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fcaa8b9d74a4b9f994bb576114fa7b0c, server=ba7f32532130,40747,1732491505794 in 183 msec
2024-11-24T23:38:27,634 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-24T23:38:27,634 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=fcaa8b9d74a4b9f994bb576114fa7b0c, ASSIGN in 343 msec
2024-11-24T23:38:27,634 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-24T23:38:27,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732491507634"}]},"ts":"1732491507634"}
2024-11-24T23:38:27,637 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-24T23:38:27,639 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-24T23:38:27,641 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 396 msec
2024-11-24T23:38:28,426 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:28,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:28,648 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T23:38:28,649 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,649 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,649 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,649 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,650 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,650 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,676 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,677 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,677 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,677 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,678 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,678 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,684 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,684 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,684 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:28,689 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:38:29,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:29,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:30,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:30,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:31,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:31,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:32,299 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-24T23:38:32,300 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-24T23:38:32,430 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:32,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:33,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T23:38:33,144 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-24T23:38:33,145 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:38:33,145 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-24T23:38:33,145 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-24T23:38:33,145 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-24T23:38:33,146 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:38:33,146 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-24T23:38:33,431 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:33,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:34,431 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:34,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:35,432 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:35,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:36,433 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:36,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T23:38:37,321 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-24T23:38:37,321 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-24T23:38:37,324 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:38:37,324 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:37,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c., hostname=ba7f32532130,40747,1732491505794, seqNum=2]
2024-11-24T23:38:37,333 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:38:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:38:37,339 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-24T23:38:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-24T23:38:37,341 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-24T23:38:37,342 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-24T23:38:37,434 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:37,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:37,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40747 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-24T23:38:37,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:37,502 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing fcaa8b9d74a4b9f994bb576114fa7b0c 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-24T23:38:37,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/e6e6577e9c4c4f19ab86d9adbd02679f is 1080, key is row0001/info:/1732491517327/Put/seqid=0
2024-11-24T23:38:37,527 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741837_1013 (size=6033)
2024-11-24T23:38:37,527 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741837_1013 (size=6033)
2024-11-24T23:38:37,528 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/e6e6577e9c4c4f19ab86d9adbd02679f
2024-11-24T23:38:37,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/e6e6577e9c4c4f19ab86d9adbd02679f as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e6e6577e9c4c4f19ab86d9adbd02679f
2024-11-24T23:38:37,544 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e6e6577e9c4c4f19ab86d9adbd02679f, entries=1, sequenceid=5, filesize=5.9 K
2024-11-24T23:38:37,545 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for fcaa8b9d74a4b9f994bb576114fa7b0c in 43ms, sequenceid=5, compaction requested=false
2024-11-24T23:38:37,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for fcaa8b9d74a4b9f994bb576114fa7b0c:

2024-11-24T23:38:37,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:37,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-24T23:38:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-24T23:38:37,553 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-24T23:38:37,553 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 209 msec
2024-11-24T23:38:37,557 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 220 msec
2024-11-24T23:38:38,435 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:38,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:39,435 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:39,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:40,436 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:40,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:41,437 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:41,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:42,438 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:42,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:43,439 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:43,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:44,440 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:44,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:45,441 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:45,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:46,442 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:46,442 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238 after 68074ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:46,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:46,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta after 68056ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T23:38:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-24T23:38:47,373 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-24T23:38:47,380 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:38:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:38:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-24T23:38:47,385 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-24T23:38:47,387 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-24T23:38:47,387 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-24T23:38:47,443 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:47,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:47,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40747 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-24T23:38:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:47,545 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing fcaa8b9d74a4b9f994bb576114fa7b0c 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-24T23:38:47,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/c385ccd8d7dc4e6a929894190281113f is 1080, key is row0002/info:/1732491527375/Put/seqid=0
2024-11-24T23:38:47,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741838_1014 (size=6033)
2024-11-24T23:38:47,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741838_1014 (size=6033)
2024-11-24T23:38:47,559 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/c385ccd8d7dc4e6a929894190281113f
2024-11-24T23:38:47,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/c385ccd8d7dc4e6a929894190281113f as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/c385ccd8d7dc4e6a929894190281113f
2024-11-24T23:38:47,574 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/c385ccd8d7dc4e6a929894190281113f, entries=1, sequenceid=9, filesize=5.9 K
2024-11-24T23:38:47,576 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for fcaa8b9d74a4b9f994bb576114fa7b0c in 30ms, sequenceid=9, compaction requested=false
2024-11-24T23:38:47,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for fcaa8b9d74a4b9f994bb576114fa7b0c:

2024-11-24T23:38:47,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:47,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-24T23:38:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-24T23:38:47,581 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-24T23:38:47,581 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 191 msec
2024-11-24T23:38:47,583 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 201 msec
2024-11-24T23:38:48,444 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:48,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:49,445 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:49,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:50,446 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:50,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:51,447 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:51,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:52,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:52,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:53,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:53,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:54,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:54,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:55,343 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T23:38:55,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:55,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:56,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:56,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:57,453 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-24T23:38:57,483 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-24T23:38:57,487 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C40747%2C1732491505794.1732491537487
2024-11-24T23:38:57,494 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:57,494 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:57,495 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:57,495 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:57,495 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:38:57,495 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491506452 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491537487
2024-11-24T23:38:57,496 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45075:45075),(127.0.0.1/127.0.0.1:33113:33113)]
2024-11-24T23:38:57,496 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491506452 is not closed yet, will try archiving it next time
2024-11-24T23:38:57,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741833_1009 (size=5546)
2024-11-24T23:38:57,498 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:38:57,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741833_1009 (size=5546)
2024-11-24T23:38:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:38:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-24T23:38:57,501 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-24T23:38:57,502 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-24T23:38:57,502 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-24T23:38:57,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:57,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40747 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-24T23:38:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:57,658 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing fcaa8b9d74a4b9f994bb576114fa7b0c 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-24T23:38:57,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/e9322f6749f74ff2b7497bff454b8bcd is 1080, key is row0003/info:/1732491537485/Put/seqid=0
2024-11-24T23:38:57,672 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741840_1016 (size=6033)
2024-11-24T23:38:57,672 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741840_1016 (size=6033)
2024-11-24T23:38:57,673 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/e9322f6749f74ff2b7497bff454b8bcd
2024-11-24T23:38:57,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/e9322f6749f74ff2b7497bff454b8bcd as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e9322f6749f74ff2b7497bff454b8bcd
2024-11-24T23:38:57,689 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e9322f6749f74ff2b7497bff454b8bcd, entries=1, sequenceid=13, filesize=5.9 K
2024-11-24T23:38:57,690 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for fcaa8b9d74a4b9f994bb576114fa7b0c in 32ms, sequenceid=13, compaction requested=true
2024-11-24T23:38:57,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for fcaa8b9d74a4b9f994bb576114fa7b0c:

2024-11-24T23:38:57,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:38:57,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-24T23:38:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-24T23:38:57,694 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-24T23:38:57,694 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 190 msec
2024-11-24T23:38:57,696 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 197 msec
2024-11-24T23:38:57,900 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491506452 to hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/oldWALs/ba7f32532130%2C40747%2C1732491505794.1732491506452
2024-11-24T23:38:58,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:58,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:59,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:38:59,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:00,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:00,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:01,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:01,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:02,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:02,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:03,459 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:03,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:04,460 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:04,516 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:05,461 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:05,516 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:06,462 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:06,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:07,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:07,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-24T23:39:07,581 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-24T23:39:07,581 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:07,583 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:39:07,583 DEBUG [Time-limited test {}] regionserver.HStore(1541): fcaa8b9d74a4b9f994bb576114fa7b0c/info is initiating minor compaction (all files)
2024-11-24T23:39:07,583 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T23:39:07,583 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:07,583 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of fcaa8b9d74a4b9f994bb576114fa7b0c/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:39:07,583 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e6e6577e9c4c4f19ab86d9adbd02679f, hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/c385ccd8d7dc4e6a929894190281113f, hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e9322f6749f74ff2b7497bff454b8bcd] into tmpdir=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp, totalSize=17.7 K
2024-11-24T23:39:07,583 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e6e6577e9c4c4f19ab86d9adbd02679f, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732491517327
2024-11-24T23:39:07,584 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c385ccd8d7dc4e6a929894190281113f, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1732491527375
2024-11-24T23:39:07,584 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e9322f6749f74ff2b7497bff454b8bcd, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732491537485
2024-11-24T23:39:07,595 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): fcaa8b9d74a4b9f994bb576114fa7b0c#info#compaction#45 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:07,596 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/c697abfaf51147239b9cc84ff0bb1ed1 is 1080, key is row0001/info:/1732491517327/Put/seqid=0
2024-11-24T23:39:07,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741841_1017 (size=8296)
2024-11-24T23:39:07,601 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741841_1017 (size=8296)
2024-11-24T23:39:07,607 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/c697abfaf51147239b9cc84ff0bb1ed1 as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/c697abfaf51147239b9cc84ff0bb1ed1
2024-11-24T23:39:07,614 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fcaa8b9d74a4b9f994bb576114fa7b0c/info of fcaa8b9d74a4b9f994bb576114fa7b0c into c697abfaf51147239b9cc84ff0bb1ed1(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:07,614 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for fcaa8b9d74a4b9f994bb576114fa7b0c:

2024-11-24T23:39:07,617 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C40747%2C1732491505794.1732491547616
2024-11-24T23:39:07,622 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:07,622 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:07,622 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:07,622 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:07,622 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:07,623 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491537487 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491547616
2024-11-24T23:39:07,623 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33113:33113),(127.0.0.1/127.0.0.1:45075:45075)]
2024-11-24T23:39:07,623 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491537487 is not closed yet, will try archiving it next time
2024-11-24T23:39:07,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741839_1015 (size=2520)
2024-11-24T23:39:07,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741839_1015 (size=2520)
2024-11-24T23:39:07,624 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:39:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:39:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-24T23:39:07,627 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-24T23:39:07,628 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-24T23:39:07,628 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-24T23:39:07,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40747 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-24T23:39:07,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:39:07,781 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing fcaa8b9d74a4b9f994bb576114fa7b0c 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-24T23:39:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/8734d9561ffc41be9f3e3943ee45f4e7 is 1080, key is row0000/info:/1732491547615/Put/seqid=0
2024-11-24T23:39:07,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741843_1019 (size=6033)
2024-11-24T23:39:07,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741843_1019 (size=6033)
2024-11-24T23:39:07,791 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/8734d9561ffc41be9f3e3943ee45f4e7
2024-11-24T23:39:07,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/8734d9561ffc41be9f3e3943ee45f4e7 as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/8734d9561ffc41be9f3e3943ee45f4e7
2024-11-24T23:39:07,802 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/8734d9561ffc41be9f3e3943ee45f4e7, entries=1, sequenceid=18, filesize=5.9 K
2024-11-24T23:39:07,803 INFO  [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for fcaa8b9d74a4b9f994bb576114fa7b0c in 22ms, sequenceid=18, compaction requested=false
2024-11-24T23:39:07,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for fcaa8b9d74a4b9f994bb576114fa7b0c:

2024-11-24T23:39:07,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:39:07,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-24T23:39:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-24T23:39:07,807 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-24T23:39:07,807 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec
2024-11-24T23:39:07,810 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec
2024-11-24T23:39:08,003 INFO  [master/ba7f32532130:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-24T23:39:08,003 INFO  [master/ba7f32532130:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-24T23:39:08,464 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:08,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:09,465 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:09,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:10,466 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:10,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:11,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:11,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:12,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:12,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:12,600 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region fcaa8b9d74a4b9f994bb576114fa7b0c, had cached 0 bytes from a total of 14329
2024-11-24T23:39:13,468 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:13,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:14,469 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:14,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:15,469 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:15,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:16,471 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:16,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:17,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:17,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35219 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-24T23:39:17,693 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-24T23:39:17,700 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C40747%2C1732491505794.1732491557699
2024-11-24T23:39:17,709 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:17,709 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:17,709 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:17,709 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:17,709 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:17,710 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491547616 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491557699
2024-11-24T23:39:17,710 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33113:33113),(127.0.0.1/127.0.0.1:45075:45075)]
2024-11-24T23:39:17,710 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491547616 is not closed yet, will try archiving it next time
2024-11-24T23:39:17,710 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491537487 to hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/oldWALs/ba7f32532130%2C40747%2C1732491505794.1732491537487
2024-11-24T23:39:17,710 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T23:39:17,711 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:39:17,711 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:39:17,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:39:17,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:39:17,711 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T23:39:17,711 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=135522427, stopped=false
2024-11-24T23:39:17,711 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ba7f32532130,35219,1732491505374
2024-11-24T23:39:17,712 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T23:39:17,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741842_1018 (size=2026)
2024-11-24T23:39:17,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741842_1018 (size=2026)
2024-11-24T23:39:17,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:39:17,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:39:17,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:17,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:17,789 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:39:17,790 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:39:17,790 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:39:17,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:39:17,791 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:39:17,791 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba7f32532130,40747,1732491505794' *****
2024-11-24T23:39:17,792 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T23:39:17,791 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:39:17,792 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T23:39:17,792 INFO  [RS:0;ba7f32532130:40747 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T23:39:17,792 INFO  [RS:0;ba7f32532130:40747 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T23:39:17,792 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T23:39:17,793 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(3091): Received CLOSE for fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:39:17,793 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(959): stopping server ba7f32532130,40747,1732491505794
2024-11-24T23:39:17,793 INFO  [RS:0;ba7f32532130:40747 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:39:17,794 INFO  [RS:0;ba7f32532130:40747 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ba7f32532130:40747.
2024-11-24T23:39:17,794 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fcaa8b9d74a4b9f994bb576114fa7b0c, disabling compactions & flushes
2024-11-24T23:39:17,794 DEBUG [RS:0;ba7f32532130:40747 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:39:17,794 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:39:17,794 DEBUG [RS:0;ba7f32532130:40747 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:39:17,794 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:39:17,794 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c. after waiting 0 ms
2024-11-24T23:39:17,794 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T23:39:17,794 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:39:17,794 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T23:39:17,794 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T23:39:17,794 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T23:39:17,794 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fcaa8b9d74a4b9f994bb576114fa7b0c 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-24T23:39:17,795 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-24T23:39:17,795 DEBUG [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, fcaa8b9d74a4b9f994bb576114fa7b0c=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.}
2024-11-24T23:39:17,795 DEBUG [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, fcaa8b9d74a4b9f994bb576114fa7b0c
2024-11-24T23:39:17,795 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:39:17,795 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:39:17,795 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:39:17,795 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:39:17,796 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:39:17,796 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-24T23:39:17,800 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/1073216c388a41c4a7d14112e79cf6d1 is 1080, key is row0001/info:/1732491557696/Put/seqid=0
2024-11-24T23:39:17,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741845_1021 (size=6033)
2024-11-24T23:39:17,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741845_1021 (size=6033)
2024-11-24T23:39:17,807 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/1073216c388a41c4a7d14112e79cf6d1
2024-11-24T23:39:17,813 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/.tmp/info/1073216c388a41c4a7d14112e79cf6d1 as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/1073216c388a41c4a7d14112e79cf6d1
2024-11-24T23:39:17,813 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/.tmp/info/5deb7c8afed742dbaa66f1d4f7c4e0e2 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c./info:regioninfo/1732491507616/Put/seqid=0
2024-11-24T23:39:17,818 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/1073216c388a41c4a7d14112e79cf6d1, entries=1, sequenceid=22, filesize=5.9 K
2024-11-24T23:39:17,820 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for fcaa8b9d74a4b9f994bb576114fa7b0c in 25ms, sequenceid=22, compaction requested=true
2024-11-24T23:39:17,822 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e6e6577e9c4c4f19ab86d9adbd02679f, hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/c385ccd8d7dc4e6a929894190281113f, hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e9322f6749f74ff2b7497bff454b8bcd] to archive
2024-11-24T23:39:17,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741846_1022 (size=7308)
2024-11-24T23:39:17,823 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T23:39:17,824 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741846_1022 (size=7308)
2024-11-24T23:39:17,824 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/.tmp/info/5deb7c8afed742dbaa66f1d4f7c4e0e2
2024-11-24T23:39:17,825 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e6e6577e9c4c4f19ab86d9adbd02679f to hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e6e6577e9c4c4f19ab86d9adbd02679f
2024-11-24T23:39:17,827 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/c385ccd8d7dc4e6a929894190281113f to hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/c385ccd8d7dc4e6a929894190281113f
2024-11-24T23:39:17,828 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e9322f6749f74ff2b7497bff454b8bcd to hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/info/e9322f6749f74ff2b7497bff454b8bcd
2024-11-24T23:39:17,828 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ba7f32532130:35219 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-24T23:39:17,829 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [e6e6577e9c4c4f19ab86d9adbd02679f=6033, c385ccd8d7dc4e6a929894190281113f=6033, e9322f6749f74ff2b7497bff454b8bcd=6033]
2024-11-24T23:39:17,832 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/fcaa8b9d74a4b9f994bb576114fa7b0c/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-24T23:39:17,833 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:39:17,833 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fcaa8b9d74a4b9f994bb576114fa7b0c:
Waiting for close lock at 1732491557794Running coprocessor pre-close hooks at 1732491557794Disabling compacts and flushes for region at 1732491557794Disabling writes for close at 1732491557794Obtaining lock to block concurrent updates at 1732491557794Preparing flush snapshotting stores in fcaa8b9d74a4b9f994bb576114fa7b0c at 1732491557794Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732491557795 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c. at 1732491557796 (+1 ms)Flushing fcaa8b9d74a4b9f994bb576114fa7b0c/info: creating writer at 1732491557796Flushing fcaa8b9d74a4b9f994bb576114fa7b0c/info: appending metadata at 1732491557799 (+3 ms)Flushing fcaa8b9d74a4b9f994bb576114fa7b0c/info: closing flushed file at 1732491557799Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36433d22: reopening flushed file at 1732491557812 (+13 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for fcaa8b9d74a4b9f994bb576114fa7b0c in 25ms, sequenceid=22, compaction requested=true at 1732491557820 (+8 ms)Writing region close event to WAL at 1732491557829 (+9 ms)Running coprocessor post-close hooks at 1732491557833 (+4 ms)Closed at 1732491557833
2024-11-24T23:39:17,833 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732491507243.fcaa8b9d74a4b9f994bb576114fa7b0c.
2024-11-24T23:39:17,842 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/.tmp/ns/3f25d8ce38b64524a98868c69ada5cfb is 43, key is default/ns:d/1732491507138/Put/seqid=0
2024-11-24T23:39:17,846 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741847_1023 (size=5153)
2024-11-24T23:39:17,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741847_1023 (size=5153)
2024-11-24T23:39:17,847 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/.tmp/ns/3f25d8ce38b64524a98868c69ada5cfb
2024-11-24T23:39:17,865 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/.tmp/table/558fcb861bcf470eaa48ed777a57f49e is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1732491507634/Put/seqid=0
2024-11-24T23:39:17,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741848_1024 (size=5508)
2024-11-24T23:39:17,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741848_1024 (size=5508)
2024-11-24T23:39:17,870 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/.tmp/table/558fcb861bcf470eaa48ed777a57f49e
2024-11-24T23:39:17,875 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/.tmp/info/5deb7c8afed742dbaa66f1d4f7c4e0e2 as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/info/5deb7c8afed742dbaa66f1d4f7c4e0e2
2024-11-24T23:39:17,882 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/info/5deb7c8afed742dbaa66f1d4f7c4e0e2, entries=10, sequenceid=11, filesize=7.1 K
2024-11-24T23:39:17,883 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/.tmp/ns/3f25d8ce38b64524a98868c69ada5cfb as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/ns/3f25d8ce38b64524a98868c69ada5cfb
2024-11-24T23:39:17,890 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/ns/3f25d8ce38b64524a98868c69ada5cfb, entries=2, sequenceid=11, filesize=5.0 K
2024-11-24T23:39:17,890 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/.tmp/table/558fcb861bcf470eaa48ed777a57f49e as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/table/558fcb861bcf470eaa48ed777a57f49e
2024-11-24T23:39:17,896 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/table/558fcb861bcf470eaa48ed777a57f49e, entries=2, sequenceid=11, filesize=5.4 K
2024-11-24T23:39:17,897 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=11, compaction requested=false
2024-11-24T23:39:17,901 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-24T23:39:17,902 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:39:17,902 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:39:17,902 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491557795Running coprocessor pre-close hooks at 1732491557795Disabling compacts and flushes for region at 1732491557795Disabling writes for close at 1732491557795Obtaining lock to block concurrent updates at 1732491557796 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732491557796Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1732491557796Flushing stores of hbase:meta,,1.1588230740 at 1732491557797 (+1 ms)Flushing 1588230740/info: creating writer at 1732491557797Flushing 1588230740/info: appending metadata at 1732491557813 (+16 ms)Flushing 1588230740/info: closing flushed file at 1732491557813Flushing 1588230740/ns: creating writer at 1732491557829 (+16 ms)Flushing 1588230740/ns: appending metadata at 1732491557842 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732491557842Flushing 1588230740/table: creating writer at 1732491557852 (+10 ms)Flushing 1588230740/table: appending metadata at 1732491557865 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732491557865Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bda14fe: reopening flushed file at 1732491557875 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7993caae: reopening flushed file at 1732491557882 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d527ce1: reopening flushed file at 1732491557890 (+8 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=11, compaction requested=false at 1732491557897 (+7 ms)Writing region close event to WAL at 1732491557898 (+1 ms)Running coprocessor post-close hooks at 1732491557902 (+4 ms)Closed at 1732491557902
2024-11-24T23:39:17,902 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T23:39:17,995 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(976): stopping server ba7f32532130,40747,1732491505794; all regions closed.
2024-11-24T23:39:17,997 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:17,997 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:17,997 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:17,998 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:17,998 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:18,002 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741834_1010 (size=3306)
2024-11-24T23:39:18,002 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741834_1010 (size=3306)
2024-11-24T23:39:18,006 DEBUG [RS:0;ba7f32532130:40747 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/oldWALs
2024-11-24T23:39:18,006 INFO  [RS:0;ba7f32532130:40747 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C40747%2C1732491505794.meta:.meta(num 1732491507041)
2024-11-24T23:39:18,006 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:18,006 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:18,007 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:18,007 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:18,007 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:18,009 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741844_1020 (size=1252)
2024-11-24T23:39:18,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741844_1020 (size=1252)
2024-11-24T23:39:18,113 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/WALs/ba7f32532130,40747,1732491505794/ba7f32532130%2C40747%2C1732491505794.1732491547616 to hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/oldWALs/ba7f32532130%2C40747%2C1732491505794.1732491547616
2024-11-24T23:39:18,119 DEBUG [RS:0;ba7f32532130:40747 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/oldWALs
2024-11-24T23:39:18,119 INFO  [RS:0;ba7f32532130:40747 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C40747%2C1732491505794:(num 1732491557699)
2024-11-24T23:39:18,119 DEBUG [RS:0;ba7f32532130:40747 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:39:18,119 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:39:18,120 INFO  [RS:0;ba7f32532130:40747 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:39:18,120 INFO  [RS:0;ba7f32532130:40747 {}] hbase.ChoreService(370): Chore service for: regionserver/ba7f32532130:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-24T23:39:18,120 INFO  [RS:0;ba7f32532130:40747 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:39:18,120 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:39:18,121 INFO  [RS:0;ba7f32532130:40747 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40747
2024-11-24T23:39:18,134 INFO  [RS:0;ba7f32532130:40747 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:39:18,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:39:18,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba7f32532130,40747,1732491505794
2024-11-24T23:39:18,146 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba7f32532130,40747,1732491505794]
2024-11-24T23:39:18,231 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba7f32532130,40747,1732491505794 already deleted, retry=false
2024-11-24T23:39:18,231 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba7f32532130,40747,1732491505794 expired; onlineServers=0
2024-11-24T23:39:18,231 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ba7f32532130,35219,1732491505374' *****
2024-11-24T23:39:18,231 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T23:39:18,231 INFO  [M:0;ba7f32532130:35219 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:39:18,232 INFO  [M:0;ba7f32532130:35219 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:39:18,232 DEBUG [M:0;ba7f32532130:35219 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T23:39:18,232 DEBUG [M:0;ba7f32532130:35219 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T23:39:18,232 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T23:39:18,232 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491506207 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491506207,5,FailOnTimeoutGroup]
2024-11-24T23:39:18,232 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491506216 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491506216,5,FailOnTimeoutGroup]
2024-11-24T23:39:18,233 INFO  [M:0;ba7f32532130:35219 {}] hbase.ChoreService(370): Chore service for: master/ba7f32532130:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T23:39:18,233 INFO  [M:0;ba7f32532130:35219 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:39:18,233 DEBUG [M:0;ba7f32532130:35219 {}] master.HMaster(1795): Stopping service threads
2024-11-24T23:39:18,233 INFO  [M:0;ba7f32532130:35219 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T23:39:18,233 INFO  [M:0;ba7f32532130:35219 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:39:18,233 INFO  [M:0;ba7f32532130:35219 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T23:39:18,234 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T23:39:18,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T23:39:18,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:18,241 DEBUG [M:0;ba7f32532130:35219 {}] zookeeper.ZKUtil(347): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T23:39:18,241 WARN  [M:0;ba7f32532130:35219 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T23:39:18,242 INFO  [M:0;ba7f32532130:35219 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/.lastflushedseqids
2024-11-24T23:39:18,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:39:18,246 INFO  [RS:0;ba7f32532130:40747 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:39:18,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40747-0x1016f366a6e0001, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:39:18,247 INFO  [RS:0;ba7f32532130:40747 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba7f32532130,40747,1732491505794; zookeeper connection closed.
2024-11-24T23:39:18,247 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@583354e5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@583354e5
2024-11-24T23:39:18,247 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T23:39:18,252 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741849_1025 (size=130)
2024-11-24T23:39:18,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741849_1025 (size=130)
2024-11-24T23:39:18,253 INFO  [M:0;ba7f32532130:35219 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T23:39:18,254 INFO  [M:0;ba7f32532130:35219 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T23:39:18,254 DEBUG [M:0;ba7f32532130:35219 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:39:18,254 INFO  [M:0;ba7f32532130:35219 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:18,254 DEBUG [M:0;ba7f32532130:35219 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:18,254 DEBUG [M:0;ba7f32532130:35219 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:39:18,254 DEBUG [M:0;ba7f32532130:35219 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:18,254 INFO  [M:0;ba7f32532130:35219 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.58 KB heapSize=54.99 KB
2024-11-24T23:39:18,272 DEBUG [M:0;ba7f32532130:35219 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc2eff0bd4aa41159bccbcbecc39fc72 is 82, key is hbase:meta,,1/info:regioninfo/1732491507067/Put/seqid=0
2024-11-24T23:39:18,276 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741850_1026 (size=5672)
2024-11-24T23:39:18,276 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741850_1026 (size=5672)
2024-11-24T23:39:18,277 INFO  [M:0;ba7f32532130:35219 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc2eff0bd4aa41159bccbcbecc39fc72
2024-11-24T23:39:18,297 DEBUG [M:0;ba7f32532130:35219 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/da04edcdb5fd43ceaa44b7ce7c3aa063 is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732491507641/Put/seqid=0
2024-11-24T23:39:18,302 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741851_1027 (size=7822)
2024-11-24T23:39:18,302 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741851_1027 (size=7822)
2024-11-24T23:39:18,303 INFO  [M:0;ba7f32532130:35219 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.98 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/da04edcdb5fd43ceaa44b7ce7c3aa063
2024-11-24T23:39:18,308 INFO  [M:0;ba7f32532130:35219 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for da04edcdb5fd43ceaa44b7ce7c3aa063
2024-11-24T23:39:18,319 INFO  [regionserver/ba7f32532130:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:39:18,322 DEBUG [M:0;ba7f32532130:35219 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a9ff1f813aca407098d2b6749103dc7b is 69, key is ba7f32532130,40747,1732491505794/rs:state/1732491506274/Put/seqid=0
2024-11-24T23:39:18,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741852_1028 (size=5156)
2024-11-24T23:39:18,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741852_1028 (size=5156)
2024-11-24T23:39:18,327 INFO  [M:0;ba7f32532130:35219 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a9ff1f813aca407098d2b6749103dc7b
2024-11-24T23:39:18,345 DEBUG [M:0;ba7f32532130:35219 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6a1e3aa940d24fc5a373c56e968f03f9 is 52, key is load_balancer_on/state:d/1732491507238/Put/seqid=0
2024-11-24T23:39:18,349 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741853_1029 (size=5056)
2024-11-24T23:39:18,349 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741853_1029 (size=5056)
2024-11-24T23:39:18,349 INFO  [M:0;ba7f32532130:35219 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6a1e3aa940d24fc5a373c56e968f03f9
2024-11-24T23:39:18,354 DEBUG [M:0;ba7f32532130:35219 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc2eff0bd4aa41159bccbcbecc39fc72 as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc2eff0bd4aa41159bccbcbecc39fc72
2024-11-24T23:39:18,359 INFO  [M:0;ba7f32532130:35219 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc2eff0bd4aa41159bccbcbecc39fc72, entries=8, sequenceid=121, filesize=5.5 K
2024-11-24T23:39:18,360 DEBUG [M:0;ba7f32532130:35219 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/da04edcdb5fd43ceaa44b7ce7c3aa063 as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/da04edcdb5fd43ceaa44b7ce7c3aa063
2024-11-24T23:39:18,364 INFO  [M:0;ba7f32532130:35219 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for da04edcdb5fd43ceaa44b7ce7c3aa063
2024-11-24T23:39:18,364 INFO  [M:0;ba7f32532130:35219 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/da04edcdb5fd43ceaa44b7ce7c3aa063, entries=14, sequenceid=121, filesize=7.6 K
2024-11-24T23:39:18,366 DEBUG [M:0;ba7f32532130:35219 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a9ff1f813aca407098d2b6749103dc7b as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a9ff1f813aca407098d2b6749103dc7b
2024-11-24T23:39:18,370 INFO  [M:0;ba7f32532130:35219 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a9ff1f813aca407098d2b6749103dc7b, entries=1, sequenceid=121, filesize=5.0 K
2024-11-24T23:39:18,371 DEBUG [M:0;ba7f32532130:35219 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6a1e3aa940d24fc5a373c56e968f03f9 as hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6a1e3aa940d24fc5a373c56e968f03f9
2024-11-24T23:39:18,377 INFO  [M:0;ba7f32532130:35219 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33195/user/jenkins/test-data/64c1fa46-f11d-791c-8ca4-a8b139c4cb83/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6a1e3aa940d24fc5a373c56e968f03f9, entries=1, sequenceid=121, filesize=4.9 K
2024-11-24T23:39:18,378 INFO  [M:0;ba7f32532130:35219 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.58 KB/44629, heapSize ~54.93 KB/56248, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=121, compaction requested=false
2024-11-24T23:39:18,379 INFO  [M:0;ba7f32532130:35219 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:18,379 DEBUG [M:0;ba7f32532130:35219 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491558254Disabling compacts and flushes for region at 1732491558254Disabling writes for close at 1732491558254Obtaining lock to block concurrent updates at 1732491558254Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732491558254Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44629, getHeapSize=56248, getOffHeapSize=0, getCellsCount=140 at 1732491558254Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732491558255 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732491558255Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732491558271 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732491558272 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732491558281 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732491558297 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732491558297Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732491558308 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732491558322 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732491558322Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732491558331 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732491558344 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732491558344Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32158a4f: reopening flushed file at 1732491558353 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10cb675c: reopening flushed file at 1732491558359 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3be5a30f: reopening flushed file at 1732491558365 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@119663e8: reopening flushed file at 1732491558370 (+5 ms)Finished flush of dataSize ~43.58 KB/44629, heapSize ~54.93 KB/56248, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=121, compaction requested=false at 1732491558378 (+8 ms)Writing region close event to WAL at 1732491558379 (+1 ms)Closed at 1732491558379
2024-11-24T23:39:18,379 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:18,379 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:18,380 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:18,380 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:18,380 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:18,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44333 is added to blk_1073741830_1006 (size=53026)
2024-11-24T23:39:18,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40221 is added to blk_1073741830_1006 (size=53026)
2024-11-24T23:39:18,382 INFO  [M:0;ba7f32532130:35219 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T23:39:18,382 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:39:18,382 INFO  [M:0;ba7f32532130:35219 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35219
2024-11-24T23:39:18,382 INFO  [M:0;ba7f32532130:35219 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:39:18,473 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:18,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:18,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:39:18,534 INFO  [M:0;ba7f32532130:35219 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:39:18,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35219-0x1016f366a6e0000, quorum=127.0.0.1:53988, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:39:18,536 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c6abbb8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:39:18,537 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@555a4a92{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:39:18,537 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:39:18,537 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f411ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:39:18,537 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ec76923{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/hadoop.log.dir/,STOPPED}
2024-11-24T23:39:18,538 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:39:18,538 WARN  [BP-324770769-172.17.0.3-1732491503088 heartbeating to localhost/127.0.0.1:33195 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:39:18,538 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:39:18,538 WARN  [BP-324770769-172.17.0.3-1732491503088 heartbeating to localhost/127.0.0.1:33195 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-324770769-172.17.0.3-1732491503088 (Datanode Uuid 81a40776-8776-4730-953a-9a71b59ec274) service to localhost/127.0.0.1:33195
2024-11-24T23:39:18,539 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f/data/data3/current/BP-324770769-172.17.0.3-1732491503088 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:39:18,539 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f/data/data4/current/BP-324770769-172.17.0.3-1732491503088 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:39:18,540 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:39:18,541 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5127cbf0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:39:18,542 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@9b25e94{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:39:18,542 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:39:18,542 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74b5ebca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:39:18,542 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39f00f78{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/hadoop.log.dir/,STOPPED}
2024-11-24T23:39:18,543 WARN  [BP-324770769-172.17.0.3-1732491503088 heartbeating to localhost/127.0.0.1:33195 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:39:18,543 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:39:18,543 WARN  [BP-324770769-172.17.0.3-1732491503088 heartbeating to localhost/127.0.0.1:33195 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-324770769-172.17.0.3-1732491503088 (Datanode Uuid 4bcc870c-3aa6-4c01-b5e1-759d40dfd57e) service to localhost/127.0.0.1:33195
2024-11-24T23:39:18,543 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:39:18,544 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f/data/data1/current/BP-324770769-172.17.0.3-1732491503088 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:39:18,544 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/cluster_228e3da9-b95d-2030-3697-3a2aa745ba9f/data/data2/current/BP-324770769-172.17.0.3-1732491503088 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:39:18,544 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:39:18,551 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2719a663{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:39:18,552 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1af676f5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:39:18,552 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:39:18,552 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc3ea71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:39:18,552 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a4b134d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/hadoop.log.dir/,STOPPED}
2024-11-24T23:39:18,559 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T23:39:18,577 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T23:39:18,584 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=208 (was 182)
Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33195
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33195 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:33195
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33195
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33195
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33195
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:33195
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33195 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33195 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 298), ProcessCount=11 (was 11), AvailableMemoryMB=3056 (was 3310)
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=208, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=3056
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/hadoop.log.dir so I do NOT create it in target/test-data/a4546196-4221-0b78-aadc-41eef6aef455
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d14e200-e3ee-9a42-3c71-2e123dcbbc32/hadoop.tmp.dir so I do NOT create it in target/test-data/a4546196-4221-0b78-aadc-41eef6aef455
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee, deleteOnExit=true
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/test.cache.data in system properties and HBase conf
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/hadoop.log.dir in system properties and HBase conf
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T23:39:18,591 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T23:39:18,592 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/nfs.dump.dir in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/java.io.tmpdir in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:39:18,592 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T23:39:18,593 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T23:39:18,604 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:39:18,946 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:39:18,949 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:39:18,950 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:39:18,950 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:39:18,950 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:39:18,951 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:39:18,951 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d6e0cf8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:39:18,951 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2996c87f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:39:19,045 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@43fb00bc{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/java.io.tmpdir/jetty-localhost-34185-hadoop-hdfs-3_4_1-tests_jar-_-any-17063766516854055831/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:39:19,045 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3def21d3{HTTP/1.1, (http/1.1)}{localhost:34185}
2024-11-24T23:39:19,045 INFO  [Time-limited test {}] server.Server(415): Started @253629ms
2024-11-24T23:39:19,056 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:39:19,304 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:39:19,307 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:39:19,308 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:39:19,308 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:39:19,308 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:39:19,309 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7cd1acca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:39:19,309 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37b300d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:39:19,404 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@616d506d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/java.io.tmpdir/jetty-localhost-34559-hadoop-hdfs-3_4_1-tests_jar-_-any-14103110484618858662/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:39:19,404 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bb23947{HTTP/1.1, (http/1.1)}{localhost:34559}
2024-11-24T23:39:19,404 INFO  [Time-limited test {}] server.Server(415): Started @253988ms
2024-11-24T23:39:19,405 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:39:19,436 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:39:19,441 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:39:19,442 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:39:19,442 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:39:19,442 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:39:19,442 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7181dda1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:39:19,442 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47c8059{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:39:19,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:19,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:19,534 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@e483a61{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/java.io.tmpdir/jetty-localhost-42441-hadoop-hdfs-3_4_1-tests_jar-_-any-10316908685212325557/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:39:19,534 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ab86f9f{HTTP/1.1, (http/1.1)}{localhost:42441}
2024-11-24T23:39:19,534 INFO  [Time-limited test {}] server.Server(415): Started @254118ms
2024-11-24T23:39:19,535 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:39:20,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:20,527 WARN  [Thread-1960 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee/data/data1/current/BP-1650221223-172.17.0.3-1732491558615/current, will proceed with Du for space computation calculation, 
2024-11-24T23:39:20,528 WARN  [Thread-1961 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee/data/data2/current/BP-1650221223-172.17.0.3-1732491558615/current, will proceed with Du for space computation calculation, 
2024-11-24T23:39:20,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:20,543 WARN  [Thread-1924 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:39:20,545 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84455d4dfe18b2a3 with lease ID 0xe1fae3952fabb04f: Processing first storage report for DS-21b0af5b-c46d-4cdd-808c-eb367f694ad1 from datanode DatanodeRegistration(127.0.0.1:41415, datanodeUuid=72d03dd0-aca6-4793-b05e-cf31d8fc0d5e, infoPort=34627, infoSecurePort=0, ipcPort=37641, storageInfo=lv=-57;cid=testClusterID;nsid=1448479817;c=1732491558615)
2024-11-24T23:39:20,545 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84455d4dfe18b2a3 with lease ID 0xe1fae3952fabb04f: from storage DS-21b0af5b-c46d-4cdd-808c-eb367f694ad1 node DatanodeRegistration(127.0.0.1:41415, datanodeUuid=72d03dd0-aca6-4793-b05e-cf31d8fc0d5e, infoPort=34627, infoSecurePort=0, ipcPort=37641, storageInfo=lv=-57;cid=testClusterID;nsid=1448479817;c=1732491558615), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:39:20,545 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84455d4dfe18b2a3 with lease ID 0xe1fae3952fabb04f: Processing first storage report for DS-41eb031b-8f3d-43b0-b41a-d7b1ee438333 from datanode DatanodeRegistration(127.0.0.1:41415, datanodeUuid=72d03dd0-aca6-4793-b05e-cf31d8fc0d5e, infoPort=34627, infoSecurePort=0, ipcPort=37641, storageInfo=lv=-57;cid=testClusterID;nsid=1448479817;c=1732491558615)
2024-11-24T23:39:20,545 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84455d4dfe18b2a3 with lease ID 0xe1fae3952fabb04f: from storage DS-41eb031b-8f3d-43b0-b41a-d7b1ee438333 node DatanodeRegistration(127.0.0.1:41415, datanodeUuid=72d03dd0-aca6-4793-b05e-cf31d8fc0d5e, infoPort=34627, infoSecurePort=0, ipcPort=37641, storageInfo=lv=-57;cid=testClusterID;nsid=1448479817;c=1732491558615), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:39:20,676 WARN  [Thread-1971 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee/data/data3/current/BP-1650221223-172.17.0.3-1732491558615/current, will proceed with Du for space computation calculation, 
2024-11-24T23:39:20,677 WARN  [Thread-1972 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee/data/data4/current/BP-1650221223-172.17.0.3-1732491558615/current, will proceed with Du for space computation calculation, 
2024-11-24T23:39:20,693 WARN  [Thread-1947 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:39:20,695 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe6934c7fe1edc6 with lease ID 0xe1fae3952fabb050: Processing first storage report for DS-0ab3f05a-166f-4980-8a2c-a33d3be848e1 from datanode DatanodeRegistration(127.0.0.1:43417, datanodeUuid=4ea9c9fa-3b20-491b-a604-34ca24b8dbf8, infoPort=40115, infoSecurePort=0, ipcPort=36085, storageInfo=lv=-57;cid=testClusterID;nsid=1448479817;c=1732491558615)
2024-11-24T23:39:20,695 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe6934c7fe1edc6 with lease ID 0xe1fae3952fabb050: from storage DS-0ab3f05a-166f-4980-8a2c-a33d3be848e1 node DatanodeRegistration(127.0.0.1:43417, datanodeUuid=4ea9c9fa-3b20-491b-a604-34ca24b8dbf8, infoPort=40115, infoSecurePort=0, ipcPort=36085, storageInfo=lv=-57;cid=testClusterID;nsid=1448479817;c=1732491558615), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:39:20,695 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe6934c7fe1edc6 with lease ID 0xe1fae3952fabb050: Processing first storage report for DS-8c3fa0f6-c5f4-4b3b-bb01-9f7d28a95ac2 from datanode DatanodeRegistration(127.0.0.1:43417, datanodeUuid=4ea9c9fa-3b20-491b-a604-34ca24b8dbf8, infoPort=40115, infoSecurePort=0, ipcPort=36085, storageInfo=lv=-57;cid=testClusterID;nsid=1448479817;c=1732491558615)
2024-11-24T23:39:20,695 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe6934c7fe1edc6 with lease ID 0xe1fae3952fabb050: from storage DS-8c3fa0f6-c5f4-4b3b-bb01-9f7d28a95ac2 node DatanodeRegistration(127.0.0.1:43417, datanodeUuid=4ea9c9fa-3b20-491b-a604-34ca24b8dbf8, infoPort=40115, infoSecurePort=0, ipcPort=36085, storageInfo=lv=-57;cid=testClusterID;nsid=1448479817;c=1732491558615), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:39:20,769 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455
2024-11-24T23:39:20,772 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee/zookeeper_0, clientPort=51651, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T23:39:20,773 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51651
2024-11-24T23:39:20,773 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:39:20,775 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:39:20,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:39:20,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:39:20,784 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e with version=8
2024-11-24T23:39:20,784 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/hbase-staging
2024-11-24T23:39:20,786 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:39:20,786 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:39:20,786 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:39:20,786 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:39:20,786 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:39:20,786 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:39:20,786 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T23:39:20,786 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:39:20,787 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35835
2024-11-24T23:39:20,789 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35835 connecting to ZooKeeper ensemble=127.0.0.1:51651
2024-11-24T23:39:20,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358350x0, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:39:20,874 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35835-0x1016f3742ef0000 connected
2024-11-24T23:39:20,958 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:39:20,960 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:39:20,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:39:20,963 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e, hbase.cluster.distributed=false
2024-11-24T23:39:20,966 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:39:20,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35835
2024-11-24T23:39:20,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35835
2024-11-24T23:39:20,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35835
2024-11-24T23:39:20,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35835
2024-11-24T23:39:20,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35835
2024-11-24T23:39:20,987 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:39:20,987 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:39:20,987 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:39:20,987 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:39:20,987 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:39:20,987 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:39:20,987 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T23:39:20,987 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:39:20,988 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46359
2024-11-24T23:39:20,989 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46359 connecting to ZooKeeper ensemble=127.0.0.1:51651
2024-11-24T23:39:20,989 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:39:20,991 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:39:21,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:463590x0, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:39:21,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:39:21,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46359-0x1016f3742ef0001 connected
2024-11-24T23:39:21,000 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T23:39:21,001 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T23:39:21,001 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T23:39:21,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:39:21,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46359
2024-11-24T23:39:21,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46359
2024-11-24T23:39:21,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46359
2024-11-24T23:39:21,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46359
2024-11-24T23:39:21,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46359
2024-11-24T23:39:21,015 DEBUG [M:0;ba7f32532130:35835 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ba7f32532130:35835
2024-11-24T23:39:21,015 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ba7f32532130,35835,1732491560786
2024-11-24T23:39:21,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:39:21,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:39:21,021 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ba7f32532130,35835,1732491560786
2024-11-24T23:39:21,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T23:39:21,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,032 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T23:39:21,032 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ba7f32532130,35835,1732491560786 from backup master directory
2024-11-24T23:39:21,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ba7f32532130,35835,1732491560786
2024-11-24T23:39:21,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:39:21,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:39:21,041 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:39:21,041 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ba7f32532130,35835,1732491560786
2024-11-24T23:39:21,046 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/hbase.id] with ID: 112584e2-037f-4d6b-b553-508545266896
2024-11-24T23:39:21,046 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/.tmp/hbase.id
2024-11-24T23:39:21,052 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:39:21,053 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:39:21,054 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/.tmp/hbase.id]:[hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/hbase.id]
2024-11-24T23:39:21,069 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:39:21,069 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T23:39:21,071 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-24T23:39:21,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,089 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:39:21,089 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:39:21,090 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:39:21,090 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T23:39:21,091 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:39:21,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:39:21,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:39:21,099 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store
2024-11-24T23:39:21,106 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:39:21,107 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:39:21,108 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:39:21,108 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:39:21,108 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:21,108 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:21,108 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:39:21,108 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:21,108 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:21,108 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491561108Disabling compacts and flushes for region at 1732491561108Disabling writes for close at 1732491561108Writing region close event to WAL at 1732491561108Closed at 1732491561108
2024-11-24T23:39:21,109 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/.initializing
2024-11-24T23:39:21,109 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/WALs/ba7f32532130,35835,1732491560786
2024-11-24T23:39:21,112 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C35835%2C1732491560786, suffix=, logDir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/WALs/ba7f32532130,35835,1732491560786, archiveDir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/oldWALs, maxLogs=10
2024-11-24T23:39:21,112 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C35835%2C1732491560786.1732491561112
2024-11-24T23:39:21,117 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/WALs/ba7f32532130,35835,1732491560786/ba7f32532130%2C35835%2C1732491560786.1732491561112
2024-11-24T23:39:21,124 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40115:40115),(127.0.0.1/127.0.0.1:34627:34627)]
2024-11-24T23:39:21,125 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:39:21,125 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:39:21,125 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:39:21,125 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:39:21,126 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:39:21,127 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T23:39:21,127 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,128 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:39:21,128 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:39:21,129 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T23:39:21,129 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,129 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:39:21,129 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:39:21,130 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T23:39:21,130 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,131 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:39:21,131 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:39:21,131 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T23:39:21,132 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,132 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:39:21,132 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:39:21,133 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:39:21,133 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:39:21,134 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:39:21,134 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:39:21,134 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T23:39:21,135 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:39:21,137 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:39:21,137 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=831365, jitterRate=0.05713586509227753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T23:39:21,138 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732491561125Initializing all the Stores at 1732491561126 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491561126Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491561126Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491561126Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491561126Cleaning up temporary data from old regions at 1732491561134 (+8 ms)Region opened successfully at 1732491561138 (+4 ms)
2024-11-24T23:39:21,138 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T23:39:21,141 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43a02187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:39:21,141 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T23:39:21,141 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T23:39:21,141 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T23:39:21,142 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T23:39:21,142 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T23:39:21,142 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T23:39:21,142 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T23:39:21,145 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T23:39:21,146 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T23:39:21,157 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T23:39:21,158 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T23:39:21,158 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T23:39:21,168 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T23:39:21,168 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T23:39:21,169 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T23:39:21,178 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T23:39:21,179 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T23:39:21,189 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T23:39:21,191 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T23:39:21,199 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T23:39:21,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:39:21,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:39:21,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,211 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ba7f32532130,35835,1732491560786, sessionid=0x1016f3742ef0000, setting cluster-up flag (Was=false)
2024-11-24T23:39:21,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,263 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T23:39:21,264 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,35835,1732491560786
2024-11-24T23:39:21,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,315 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T23:39:21,316 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,35835,1732491560786
2024-11-24T23:39:21,317 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T23:39:21,319 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T23:39:21,319 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T23:39:21,319 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T23:39:21,319 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ba7f32532130,35835,1732491560786
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T23:39:21,320 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:39:21,320 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:39:21,320 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:39:21,320 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:39:21,320 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ba7f32532130:0, corePoolSize=10, maxPoolSize=10
2024-11-24T23:39:21,321 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,321 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:39:21,321 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,323 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732491591323
2024-11-24T23:39:21,324 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T23:39:21,324 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T23:39:21,324 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T23:39:21,324 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T23:39:21,324 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T23:39:21,324 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T23:39:21,324 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,324 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:39:21,324 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T23:39:21,324 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T23:39:21,324 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T23:39:21,325 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T23:39:21,325 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T23:39:21,325 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T23:39:21,325 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491561325,5,FailOnTimeoutGroup]
2024-11-24T23:39:21,325 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491561325,5,FailOnTimeoutGroup]
2024-11-24T23:39:21,325 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,325 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,325 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T23:39:21,325 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,325 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,325 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T23:39:21,330 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:39:21,331 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:39:21,331 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T23:39:21,332 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e
2024-11-24T23:39:21,337 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:39:21,337 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741832_1008 (size=32)
2024-11-24T23:39:21,338 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:39:21,339 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:39:21,340 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:39:21,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,340 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:39:21,340 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:39:21,341 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:39:21,342 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,342 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:39:21,342 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:39:21,343 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:39:21,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,344 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:39:21,344 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:39:21,345 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:39:21,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,345 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:39:21,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:39:21,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740
2024-11-24T23:39:21,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740
2024-11-24T23:39:21,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:39:21,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:39:21,348 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:39:21,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:39:21,351 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:39:21,351 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=791108, jitterRate=0.005946457386016846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:39:21,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732491561338Initializing all the Stores at 1732491561339 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491561339Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491561339Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491561339Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491561339Cleaning up temporary data from old regions at 1732491561347 (+8 ms)Region opened successfully at 1732491561352 (+5 ms)
2024-11-24T23:39:21,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:39:21,352 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:39:21,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:39:21,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:39:21,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:39:21,353 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:39:21,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491561352Disabling compacts and flushes for region at 1732491561352Disabling writes for close at 1732491561352Writing region close event to WAL at 1732491561353 (+1 ms)Closed at 1732491561353
2024-11-24T23:39:21,354 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:39:21,354 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T23:39:21,354 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T23:39:21,356 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:39:21,357 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T23:39:21,406 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(746): ClusterId : 112584e2-037f-4d6b-b553-508545266896
2024-11-24T23:39:21,406 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T23:39:21,411 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T23:39:21,411 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T23:39:21,422 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T23:39:21,422 DEBUG [RS:0;ba7f32532130:46359 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44795b69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:39:21,456 DEBUG [RS:0;ba7f32532130:46359 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ba7f32532130:46359
2024-11-24T23:39:21,456 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T23:39:21,456 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T23:39:21,456 DEBUG [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T23:39:21,457 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba7f32532130,35835,1732491560786 with port=46359, startcode=1732491560986
2024-11-24T23:39:21,457 DEBUG [RS:0;ba7f32532130:46359 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T23:39:21,459 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55795, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T23:39:21,459 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35835 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba7f32532130,46359,1732491560986
2024-11-24T23:39:21,459 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35835 {}] master.ServerManager(517): Registering regionserver=ba7f32532130,46359,1732491560986
2024-11-24T23:39:21,461 DEBUG [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e
2024-11-24T23:39:21,461 DEBUG [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44631
2024-11-24T23:39:21,461 DEBUG [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T23:39:21,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:39:21,474 DEBUG [RS:0;ba7f32532130:46359 {}] zookeeper.ZKUtil(111): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba7f32532130,46359,1732491560986
2024-11-24T23:39:21,474 WARN  [RS:0;ba7f32532130:46359 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:39:21,474 INFO  [RS:0;ba7f32532130:46359 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:39:21,474 DEBUG [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986
2024-11-24T23:39:21,474 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba7f32532130,46359,1732491560986]
2024-11-24T23:39:21,475 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:21,477 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T23:39:21,479 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T23:39:21,479 INFO  [RS:0;ba7f32532130:46359 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T23:39:21,479 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,479 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T23:39:21,480 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T23:39:21,480 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:39:21,480 DEBUG [RS:0;ba7f32532130:46359 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:39:21,481 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,481 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,481 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,481 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,481 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,481 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,46359,1732491560986-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:39:21,501 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T23:39:21,501 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,46359,1732491560986-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,501 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,501 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.Replication(171): ba7f32532130,46359,1732491560986 started
2024-11-24T23:39:21,507 WARN  [ba7f32532130:35835 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T23:39:21,517 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:21,517 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(1482): Serving as ba7f32532130,46359,1732491560986, RpcServer on ba7f32532130/172.17.0.3:46359, sessionid=0x1016f3742ef0001
2024-11-24T23:39:21,517 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T23:39:21,517 DEBUG [RS:0;ba7f32532130:46359 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba7f32532130,46359,1732491560986
2024-11-24T23:39:21,517 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,46359,1732491560986'
2024-11-24T23:39:21,517 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T23:39:21,518 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T23:39:21,519 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T23:39:21,519 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T23:39:21,519 DEBUG [RS:0;ba7f32532130:46359 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba7f32532130,46359,1732491560986
2024-11-24T23:39:21,519 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,46359,1732491560986'
2024-11-24T23:39:21,519 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T23:39:21,519 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T23:39:21,519 DEBUG [RS:0;ba7f32532130:46359 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T23:39:21,519 INFO  [RS:0;ba7f32532130:46359 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T23:39:21,519 INFO  [RS:0;ba7f32532130:46359 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T23:39:21,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:21,624 INFO  [RS:0;ba7f32532130:46359 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C46359%2C1732491560986, suffix=, logDir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986, archiveDir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/oldWALs, maxLogs=32
2024-11-24T23:39:21,626 INFO  [RS:0;ba7f32532130:46359 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C46359%2C1732491560986.1732491561625
2024-11-24T23:39:21,634 INFO  [RS:0;ba7f32532130:46359 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986/ba7f32532130%2C46359%2C1732491560986.1732491561625
2024-11-24T23:39:21,635 DEBUG [RS:0;ba7f32532130:46359 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34627:34627),(127.0.0.1/127.0.0.1:40115:40115)]
2024-11-24T23:39:21,757 DEBUG [ba7f32532130:35835 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T23:39:21,757 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba7f32532130,46359,1732491560986
2024-11-24T23:39:21,759 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,46359,1732491560986, state=OPENING
2024-11-24T23:39:21,768 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T23:39:21,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:21,779 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:39:21,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:39:21,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:39:21,779 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,46359,1732491560986}]
2024-11-24T23:39:21,934 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T23:39:21,940 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46685, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T23:39:21,945 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T23:39:21,945 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:39:21,947 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C46359%2C1732491560986.meta, suffix=.meta, logDir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986, archiveDir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/oldWALs, maxLogs=32
2024-11-24T23:39:21,948 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C46359%2C1732491560986.meta.1732491561947.meta
2024-11-24T23:39:21,953 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986/ba7f32532130%2C46359%2C1732491560986.meta.1732491561947.meta
2024-11-24T23:39:21,954 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34627:34627),(127.0.0.1/127.0.0.1:40115:40115)]
2024-11-24T23:39:21,955 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:39:21,955 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T23:39:21,955 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T23:39:21,955 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T23:39:21,956 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T23:39:21,956 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:39:21,956 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T23:39:21,956 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T23:39:21,957 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:39:21,958 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:39:21,958 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,959 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:39:21,959 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:39:21,960 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:39:21,960 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,960 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:39:21,960 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:39:21,961 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:39:21,961 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,961 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:39:21,961 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:39:21,962 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:39:21,962 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:21,962 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:39:21,962 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:39:21,963 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740
2024-11-24T23:39:21,964 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740
2024-11-24T23:39:21,965 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:39:21,965 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:39:21,966 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:39:21,967 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:39:21,967 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=876112, jitterRate=0.11403411626815796}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:39:21,967 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T23:39:21,968 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732491561956Writing region info on filesystem at 1732491561956Initializing all the Stores at 1732491561957 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491561957Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491561957Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491561957Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491561957Cleaning up temporary data from old regions at 1732491561965 (+8 ms)Running coprocessor post-open hooks at 1732491561967 (+2 ms)Region opened successfully at 1732491561968 (+1 ms)
2024-11-24T23:39:21,969 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732491561933
2024-11-24T23:39:21,971 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T23:39:21,971 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T23:39:21,972 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,46359,1732491560986
2024-11-24T23:39:21,973 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,46359,1732491560986, state=OPEN
2024-11-24T23:39:22,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:39:22,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:39:22,027 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:39:22,027 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba7f32532130,46359,1732491560986
2024-11-24T23:39:22,027 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:39:22,031 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T23:39:22,031 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,46359,1732491560986 in 248 msec
2024-11-24T23:39:22,035 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T23:39:22,035 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 677 msec
2024-11-24T23:39:22,036 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:39:22,036 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T23:39:22,038 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:39:22,038 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,46359,1732491560986, seqNum=-1]
2024-11-24T23:39:22,039 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:39:22,040 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54873, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:39:22,046 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 726 msec
2024-11-24T23:39:22,046 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732491562046, completionTime=-1
2024-11-24T23:39:22,046 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T23:39:22,047 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T23:39:22,049 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T23:39:22,049 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732491622049
2024-11-24T23:39:22,049 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732491682049
2024-11-24T23:39:22,049 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-24T23:39:22,049 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35835,1732491560786-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:22,049 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35835,1732491560786-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:22,049 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35835,1732491560786-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:22,049 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ba7f32532130:35835, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:22,049 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:22,050 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:22,052 DEBUG [master/ba7f32532130:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T23:39:22,054 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.012sec
2024-11-24T23:39:22,054 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T23:39:22,054 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T23:39:22,054 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T23:39:22,054 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T23:39:22,055 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T23:39:22,055 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35835,1732491560786-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:39:22,055 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35835,1732491560786-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T23:39:22,058 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T23:39:22,059 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T23:39:22,059 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,35835,1732491560786-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:39:22,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ce088a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:39:22,107 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ba7f32532130,35835,-1 for getting cluster id
2024-11-24T23:39:22,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T23:39:22,111 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '112584e2-037f-4d6b-b553-508545266896'
2024-11-24T23:39:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T23:39:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "112584e2-037f-4d6b-b553-508545266896"

2024-11-24T23:39:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fad0aab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:39:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ba7f32532130,35835,-1]
2024-11-24T23:39:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T23:39:22,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:39:22,114 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T23:39:22,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9677fe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:39:22,115 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:39:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,46359,1732491560986, seqNum=-1]
2024-11-24T23:39:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:39:22,117 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:39:22,119 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ba7f32532130,35835,1732491560786
2024-11-24T23:39:22,120 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:39:22,122 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T23:39:22,122 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-24T23:39:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is ba7f32532130,35835,1732491560786
2024-11-24T23:39:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2b484343
2024-11-24T23:39:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-24T23:39:22,125 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-24T23:39:22,125 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35835 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-24T23:39:22,126 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35835 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-24T23:39:22,126 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35835 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:39:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35835 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-24T23:39:22,128 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-24T23:39:22,128 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:22,128 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35835 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-24T23:39:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35835 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T23:39:22,129 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-24T23:39:22,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741835_1011 (size=381)
2024-11-24T23:39:22,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741835_1011 (size=381)
2024-11-24T23:39:22,137 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 13f4128eec290306f228e54615cac449, NAME => 'TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e
2024-11-24T23:39:22,143 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741836_1012 (size=64)
2024-11-24T23:39:22,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741836_1012 (size=64)
2024-11-24T23:39:22,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:39:22,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 13f4128eec290306f228e54615cac449, disabling compactions & flushes
2024-11-24T23:39:22,144 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:22,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:22,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449. after waiting 0 ms
2024-11-24T23:39:22,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:22,144 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:22,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 13f4128eec290306f228e54615cac449:
Waiting for close lock at 1732491562144Disabling compacts and flushes for region at 1732491562144Disabling writes for close at 1732491562144Writing region close event to WAL at 1732491562144Closed at 1732491562144
2024-11-24T23:39:22,146 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-24T23:39:22,146 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732491562146"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732491562146"}]},"ts":"1732491562146"}
2024-11-24T23:39:22,149 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-24T23:39:22,150 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-24T23:39:22,150 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732491562150"}]},"ts":"1732491562150"}
2024-11-24T23:39:22,152 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-24T23:39:22,152 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=13f4128eec290306f228e54615cac449, ASSIGN}]
2024-11-24T23:39:22,154 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=13f4128eec290306f228e54615cac449, ASSIGN
2024-11-24T23:39:22,155 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=13f4128eec290306f228e54615cac449, ASSIGN; state=OFFLINE, location=ba7f32532130,46359,1732491560986; forceNewPlan=false, retain=false
2024-11-24T23:39:22,306 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=13f4128eec290306f228e54615cac449, regionState=OPENING, regionLocation=ba7f32532130,46359,1732491560986
2024-11-24T23:39:22,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=13f4128eec290306f228e54615cac449, ASSIGN because future has completed
2024-11-24T23:39:22,309 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13f4128eec290306f228e54615cac449, server=ba7f32532130,46359,1732491560986}]
2024-11-24T23:39:22,467 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:22,467 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 13f4128eec290306f228e54615cac449, NAME => 'TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:39:22,467 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 13f4128eec290306f228e54615cac449
2024-11-24T23:39:22,467 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:39:22,467 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 13f4128eec290306f228e54615cac449
2024-11-24T23:39:22,467 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 13f4128eec290306f228e54615cac449
2024-11-24T23:39:22,469 INFO  [StoreOpener-13f4128eec290306f228e54615cac449-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 13f4128eec290306f228e54615cac449 
2024-11-24T23:39:22,471 INFO  [StoreOpener-13f4128eec290306f228e54615cac449-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13f4128eec290306f228e54615cac449 columnFamilyName info
2024-11-24T23:39:22,472 DEBUG [StoreOpener-13f4128eec290306f228e54615cac449-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:22,472 INFO  [StoreOpener-13f4128eec290306f228e54615cac449-1 {}] regionserver.HStore(327): Store=13f4128eec290306f228e54615cac449/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:39:22,472 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 13f4128eec290306f228e54615cac449
2024-11-24T23:39:22,474 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449
2024-11-24T23:39:22,474 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449
2024-11-24T23:39:22,475 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 13f4128eec290306f228e54615cac449
2024-11-24T23:39:22,475 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 13f4128eec290306f228e54615cac449
2024-11-24T23:39:22,475 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:22,477 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 13f4128eec290306f228e54615cac449
2024-11-24T23:39:22,480 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:39:22,481 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 13f4128eec290306f228e54615cac449; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715038, jitterRate=-0.09078222513198853}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T23:39:22,481 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 13f4128eec290306f228e54615cac449
2024-11-24T23:39:22,482 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 13f4128eec290306f228e54615cac449:
Running coprocessor pre-open hook at 1732491562468Writing region info on filesystem at 1732491562468Initializing all the Stores at 1732491562469 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491562469Cleaning up temporary data from old regions at 1732491562475 (+6 ms)Running coprocessor post-open hooks at 1732491562481 (+6 ms)Region opened successfully at 1732491562482 (+1 ms)
2024-11-24T23:39:22,483 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449., pid=6, masterSystemTime=1732491562462
2024-11-24T23:39:22,486 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:22,486 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:22,487 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=13f4128eec290306f228e54615cac449, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,46359,1732491560986
2024-11-24T23:39:22,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13f4128eec290306f228e54615cac449, server=ba7f32532130,46359,1732491560986 because future has completed
2024-11-24T23:39:22,495 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-24T23:39:22,495 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 13f4128eec290306f228e54615cac449, server=ba7f32532130,46359,1732491560986 in 183 msec
2024-11-24T23:39:22,498 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-24T23:39:22,498 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=13f4128eec290306f228e54615cac449, ASSIGN in 343 msec
2024-11-24T23:39:22,499 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-24T23:39:22,499 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732491562499"}]},"ts":"1732491562499"}
2024-11-24T23:39:22,501 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-24T23:39:22,502 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-24T23:39:22,504 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 376 msec
2024-11-24T23:39:22,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:22,834 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,834 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,835 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,835 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,835 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,836 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,856 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,861 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,861 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:22,863 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,368 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T23:39:23,370 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,370 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,371 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,371 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,375 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,396 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,396 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,401 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,401 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,402 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,404 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:23,408 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-24T23:39:23,408 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-24T23:39:23,409 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T23:39:23,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:23,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:24,477 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:24,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:25,478 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:25,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:26,479 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:26,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:27,477 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-24T23:39:27,478 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-24T23:39:27,480 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:27,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:28,480 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:28,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:28,911 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T23:39:28,912 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,914 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,914 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,946 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,947 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,947 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,947 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,948 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,948 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,954 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,955 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,955 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:28,959 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:29,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:29,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:30,482 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:30,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:31,483 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:31,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35835 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T23:39:32,172 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-24T23:39:32,172 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-24T23:39:32,178 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-24T23:39:32,178 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449., hostname=ba7f32532130,46359,1732491560986, seqNum=2]
2024-11-24T23:39:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 13f4128eec290306f228e54615cac449
2024-11-24T23:39:32,196 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f4128eec290306f228e54615cac449 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T23:39:32,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/e6992a9ba01b4f61aaa23420b9fb88a9 is 1080, key is row0001/info:/1732491572184/Put/seqid=0
2024-11-24T23:39:32,220 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741837_1013 (size=12509)
2024-11-24T23:39:32,220 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741837_1013 (size=12509)
2024-11-24T23:39:32,221 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/e6992a9ba01b4f61aaa23420b9fb88a9
2024-11-24T23:39:32,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/e6992a9ba01b4f61aaa23420b9fb88a9 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/e6992a9ba01b4f61aaa23420b9fb88a9
2024-11-24T23:39:32,233 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/e6992a9ba01b4f61aaa23420b9fb88a9, entries=7, sequenceid=11, filesize=12.2 K
2024-11-24T23:39:32,234 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=19.96 KB/20444 for 13f4128eec290306f228e54615cac449 in 37ms, sequenceid=11, compaction requested=false
2024-11-24T23:39:32,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f4128eec290306f228e54615cac449:

2024-11-24T23:39:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 13f4128eec290306f228e54615cac449
2024-11-24T23:39:32,235 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f4128eec290306f228e54615cac449 1/1 column families, dataSize=22.07 KB heapSize=23.88 KB
2024-11-24T23:39:32,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/379d098d924f415689da01db8b119a44 is 1080, key is row0008/info:/1732491572197/Put/seqid=0
2024-11-24T23:39:32,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741838_1014 (size=27607)
2024-11-24T23:39:32,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741838_1014 (size=27607)
2024-11-24T23:39:32,244 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.07 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/379d098d924f415689da01db8b119a44
2024-11-24T23:39:32,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/379d098d924f415689da01db8b119a44 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/379d098d924f415689da01db8b119a44
2024-11-24T23:39:32,257 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/379d098d924f415689da01db8b119a44, entries=21, sequenceid=35, filesize=27.0 K
2024-11-24T23:39:32,258 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~22.07 KB/22596, heapSize ~23.86 KB/24432, currentSize=4.20 KB/4304 for 13f4128eec290306f228e54615cac449 in 23ms, sequenceid=35, compaction requested=false
2024-11-24T23:39:32,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f4128eec290306f228e54615cac449:

2024-11-24T23:39:32,258 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.2 K, sizeToCheck=16.0 K
2024-11-24T23:39:32,258 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:32,259 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/379d098d924f415689da01db8b119a44 because midkey is the same as first or last row
2024-11-24T23:39:32,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:32,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:33,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:33,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 13f4128eec290306f228e54615cac449
2024-11-24T23:39:34,257 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f4128eec290306f228e54615cac449 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T23:39:34,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/589ac39d09484770b578c2ec9fcbfe26 is 1080, key is row0029/info:/1732491572236/Put/seqid=0
2024-11-24T23:39:34,267 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741839_1015 (size=12509)
2024-11-24T23:39:34,267 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741839_1015 (size=12509)
2024-11-24T23:39:34,268 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/589ac39d09484770b578c2ec9fcbfe26
2024-11-24T23:39:34,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/589ac39d09484770b578c2ec9fcbfe26 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/589ac39d09484770b578c2ec9fcbfe26
2024-11-24T23:39:34,280 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/589ac39d09484770b578c2ec9fcbfe26, entries=7, sequenceid=45, filesize=12.2 K
2024-11-24T23:39:34,281 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 13f4128eec290306f228e54615cac449 in 24ms, sequenceid=45, compaction requested=true
2024-11-24T23:39:34,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f4128eec290306f228e54615cac449:

2024-11-24T23:39:34,282 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=51.4 K, sizeToCheck=16.0 K
2024-11-24T23:39:34,282 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:34,282 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/379d098d924f415689da01db8b119a44 because midkey is the same as first or last row
2024-11-24T23:39:34,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f4128eec290306f228e54615cac449:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:39:34,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:34,282 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:34,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 13f4128eec290306f228e54615cac449
2024-11-24T23:39:34,283 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f4128eec290306f228e54615cac449 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-24T23:39:34,283 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 52625 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:39:34,283 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1541): 13f4128eec290306f228e54615cac449/info is initiating minor compaction (all files)
2024-11-24T23:39:34,284 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f4128eec290306f228e54615cac449/info in TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:34,284 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/e6992a9ba01b4f61aaa23420b9fb88a9, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/379d098d924f415689da01db8b119a44, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/589ac39d09484770b578c2ec9fcbfe26] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp, totalSize=51.4 K
2024-11-24T23:39:34,284 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6992a9ba01b4f61aaa23420b9fb88a9, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732491572184
2024-11-24T23:39:34,285 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 379d098d924f415689da01db8b119a44, keycount=21, bloomtype=ROW, size=27.0 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1732491572197
2024-11-24T23:39:34,285 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 589ac39d09484770b578c2ec9fcbfe26, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732491572236
2024-11-24T23:39:34,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/b2c04e83db9445c0ba874d5ada27430b is 1080, key is row0036/info:/1732491574259/Put/seqid=0
2024-11-24T23:39:34,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741840_1016 (size=17894)
2024-11-24T23:39:34,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741840_1016 (size=17894)
2024-11-24T23:39:34,296 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/b2c04e83db9445c0ba874d5ada27430b
2024-11-24T23:39:34,301 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f4128eec290306f228e54615cac449#info#compaction#59 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:34,301 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/3c6475a2ed5e4d9ea826f59a3bb18536 is 1080, key is row0001/info:/1732491572184/Put/seqid=0
2024-11-24T23:39:34,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/b2c04e83db9445c0ba874d5ada27430b as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/b2c04e83db9445c0ba874d5ada27430b
2024-11-24T23:39:34,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741841_1017 (size=42824)
2024-11-24T23:39:34,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741841_1017 (size=42824)
2024-11-24T23:39:34,311 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/b2c04e83db9445c0ba874d5ada27430b, entries=12, sequenceid=60, filesize=17.5 K
2024-11-24T23:39:34,312 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=12.61 KB/12912 for 13f4128eec290306f228e54615cac449 in 29ms, sequenceid=60, compaction requested=false
2024-11-24T23:39:34,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f4128eec290306f228e54615cac449:

2024-11-24T23:39:34,312 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.9 K, sizeToCheck=16.0 K
2024-11-24T23:39:34,312 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:34,312 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/379d098d924f415689da01db8b119a44 because midkey is the same as first or last row
2024-11-24T23:39:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 13f4128eec290306f228e54615cac449
2024-11-24T23:39:34,314 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f4128eec290306f228e54615cac449 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-24T23:39:34,314 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/3c6475a2ed5e4d9ea826f59a3bb18536 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/3c6475a2ed5e4d9ea826f59a3bb18536
2024-11-24T23:39:34,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/72ed511a7a3240bb993eda41d62af8c6 is 1080, key is row0048/info:/1732491574284/Put/seqid=0
2024-11-24T23:39:34,321 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f4128eec290306f228e54615cac449/info of 13f4128eec290306f228e54615cac449 into 3c6475a2ed5e4d9ea826f59a3bb18536(size=41.8 K), total size for store is 59.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:34,321 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f4128eec290306f228e54615cac449:

2024-11-24T23:39:34,321 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449., storeName=13f4128eec290306f228e54615cac449/info, priority=13, startTime=1732491574282; duration=0sec
2024-11-24T23:39:34,321 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-24T23:39:34,321 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:34,321 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/3c6475a2ed5e4d9ea826f59a3bb18536 because midkey is the same as first or last row
2024-11-24T23:39:34,321 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-24T23:39:34,321 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:34,321 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/3c6475a2ed5e4d9ea826f59a3bb18536 because midkey is the same as first or last row
2024-11-24T23:39:34,321 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-24T23:39:34,321 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:34,322 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/3c6475a2ed5e4d9ea826f59a3bb18536 because midkey is the same as first or last row
2024-11-24T23:39:34,322 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:34,322 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f4128eec290306f228e54615cac449:info
2024-11-24T23:39:34,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741842_1018 (size=18987)
2024-11-24T23:39:34,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741842_1018 (size=18987)
2024-11-24T23:39:34,324 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/72ed511a7a3240bb993eda41d62af8c6
2024-11-24T23:39:34,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/72ed511a7a3240bb993eda41d62af8c6 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/72ed511a7a3240bb993eda41d62af8c6
2024-11-24T23:39:34,333 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/72ed511a7a3240bb993eda41d62af8c6, entries=13, sequenceid=76, filesize=18.5 K
2024-11-24T23:39:34,334 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=4.20 KB/4304 for 13f4128eec290306f228e54615cac449 in 20ms, sequenceid=76, compaction requested=true
2024-11-24T23:39:34,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f4128eec290306f228e54615cac449:

2024-11-24T23:39:34,334 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=77.8 K, sizeToCheck=16.0 K
2024-11-24T23:39:34,334 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:34,335 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/3c6475a2ed5e4d9ea826f59a3bb18536 because midkey is the same as first or last row
2024-11-24T23:39:34,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f4128eec290306f228e54615cac449:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:39:34,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:34,335 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:34,336 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 79705 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:39:34,336 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1541): 13f4128eec290306f228e54615cac449/info is initiating minor compaction (all files)
2024-11-24T23:39:34,336 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f4128eec290306f228e54615cac449/info in TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:34,336 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/3c6475a2ed5e4d9ea826f59a3bb18536, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/b2c04e83db9445c0ba874d5ada27430b, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/72ed511a7a3240bb993eda41d62af8c6] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp, totalSize=77.8 K
2024-11-24T23:39:34,337 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c6475a2ed5e4d9ea826f59a3bb18536, keycount=35, bloomtype=ROW, size=41.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732491572184
2024-11-24T23:39:34,337 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2c04e83db9445c0ba874d5ada27430b, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732491574259
2024-11-24T23:39:34,337 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72ed511a7a3240bb993eda41d62af8c6, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732491574284
2024-11-24T23:39:34,348 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f4128eec290306f228e54615cac449#info#compaction#61 average throughput is 30.78 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:34,349 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/fa04705436014aee90eed6c9a75036e6 is 1080, key is row0001/info:/1732491572184/Put/seqid=0
2024-11-24T23:39:34,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741843_1019 (size=69920)
2024-11-24T23:39:34,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741843_1019 (size=69920)
2024-11-24T23:39:34,359 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/fa04705436014aee90eed6c9a75036e6 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/fa04705436014aee90eed6c9a75036e6
2024-11-24T23:39:34,366 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f4128eec290306f228e54615cac449/info of 13f4128eec290306f228e54615cac449 into fa04705436014aee90eed6c9a75036e6(size=68.3 K), total size for store is 68.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:34,366 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f4128eec290306f228e54615cac449:

2024-11-24T23:39:34,366 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449., storeName=13f4128eec290306f228e54615cac449/info, priority=13, startTime=1732491574335; duration=0sec
2024-11-24T23:39:34,366 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.3 K, sizeToCheck=16.0 K
2024-11-24T23:39:34,366 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:34,366 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/fa04705436014aee90eed6c9a75036e6 because midkey is the same as first or last row
2024-11-24T23:39:34,366 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.3 K, sizeToCheck=16.0 K
2024-11-24T23:39:34,366 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:34,367 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/fa04705436014aee90eed6c9a75036e6 because midkey is the same as first or last row
2024-11-24T23:39:34,367 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.3 K, sizeToCheck=16.0 K
2024-11-24T23:39:34,367 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:34,367 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/fa04705436014aee90eed6c9a75036e6 because midkey is the same as first or last row
2024-11-24T23:39:34,367 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:34,367 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f4128eec290306f228e54615cac449:info
2024-11-24T23:39:34,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:34,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:35,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:35,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,338 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f4128eec290306f228e54615cac449 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T23:39:36,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/ef103b296db74fb88cfd45d1918d0aef is 1080, key is row0061/info:/1732491574315/Put/seqid=0
2024-11-24T23:39:36,349 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741844_1020 (size=12509)
2024-11-24T23:39:36,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741844_1020 (size=12509)
2024-11-24T23:39:36,350 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/ef103b296db74fb88cfd45d1918d0aef
2024-11-24T23:39:36,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/ef103b296db74fb88cfd45d1918d0aef as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/ef103b296db74fb88cfd45d1918d0aef
2024-11-24T23:39:36,362 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/ef103b296db74fb88cfd45d1918d0aef, entries=7, sequenceid=88, filesize=12.2 K
2024-11-24T23:39:36,363 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 13f4128eec290306f228e54615cac449 in 25ms, sequenceid=88, compaction requested=false
2024-11-24T23:39:36,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f4128eec290306f228e54615cac449:

2024-11-24T23:39:36,363 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=80.5 K, sizeToCheck=16.0 K
2024-11-24T23:39:36,363 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:36,363 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/fa04705436014aee90eed6c9a75036e6 because midkey is the same as first or last row
2024-11-24T23:39:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,364 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f4128eec290306f228e54615cac449 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-24T23:39:36,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/369ecb3b735b46088670bf9be95ba99f is 1080, key is row0068/info:/1732491576340/Put/seqid=0
2024-11-24T23:39:36,377 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741845_1021 (size=16817)
2024-11-24T23:39:36,377 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741845_1021 (size=16817)
2024-11-24T23:39:36,379 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/369ecb3b735b46088670bf9be95ba99f
2024-11-24T23:39:36,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/369ecb3b735b46088670bf9be95ba99f as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/369ecb3b735b46088670bf9be95ba99f
2024-11-24T23:39:36,393 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/369ecb3b735b46088670bf9be95ba99f, entries=11, sequenceid=102, filesize=16.4 K
2024-11-24T23:39:36,394 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=14.71 KB/15064 for 13f4128eec290306f228e54615cac449 in 29ms, sequenceid=102, compaction requested=true
2024-11-24T23:39:36,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f4128eec290306f228e54615cac449:

2024-11-24T23:39:36,394 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=96.9 K, sizeToCheck=16.0 K
2024-11-24T23:39:36,394 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:36,394 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/fa04705436014aee90eed6c9a75036e6 because midkey is the same as first or last row
2024-11-24T23:39:36,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13f4128eec290306f228e54615cac449:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:39:36,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:36,394 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:36,395 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 99246 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:39:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,395 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1541): 13f4128eec290306f228e54615cac449/info is initiating minor compaction (all files)
2024-11-24T23:39:36,396 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13f4128eec290306f228e54615cac449/info in TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:36,396 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13f4128eec290306f228e54615cac449 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-24T23:39:36,396 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/fa04705436014aee90eed6c9a75036e6, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/ef103b296db74fb88cfd45d1918d0aef, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/369ecb3b735b46088670bf9be95ba99f] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp, totalSize=96.9 K
2024-11-24T23:39:36,396 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa04705436014aee90eed6c9a75036e6, keycount=60, bloomtype=ROW, size=68.3 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732491572184
2024-11-24T23:39:36,397 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef103b296db74fb88cfd45d1918d0aef, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732491574315
2024-11-24T23:39:36,397 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 369ecb3b735b46088670bf9be95ba99f, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732491576340
2024-11-24T23:39:36,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/05ad70c1cf6b44e38bdeeab50a87ceeb is 1080, key is row0079/info:/1732491576365/Put/seqid=0
2024-11-24T23:39:36,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741846_1022 (size=22222)
2024-11-24T23:39:36,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741846_1022 (size=22222)
2024-11-24T23:39:36,407 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/05ad70c1cf6b44e38bdeeab50a87ceeb
2024-11-24T23:39:36,412 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13f4128eec290306f228e54615cac449#info#compaction#65 average throughput is 26.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:36,413 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/96c490f97dd042c4bdefc1a1ff160658 is 1080, key is row0001/info:/1732491572184/Put/seqid=0
2024-11-24T23:39:36,418 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741847_1023 (size=89485)
2024-11-24T23:39:36,418 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741847_1023 (size=89485)
2024-11-24T23:39:36,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/05ad70c1cf6b44e38bdeeab50a87ceeb as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/05ad70c1cf6b44e38bdeeab50a87ceeb
2024-11-24T23:39:36,425 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/96c490f97dd042c4bdefc1a1ff160658 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/96c490f97dd042c4bdefc1a1ff160658
2024-11-24T23:39:36,426 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/05ad70c1cf6b44e38bdeeab50a87ceeb, entries=16, sequenceid=121, filesize=21.7 K
2024-11-24T23:39:36,427 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=2.10 KB/2152 for 13f4128eec290306f228e54615cac449 in 32ms, sequenceid=121, compaction requested=false
2024-11-24T23:39:36,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13f4128eec290306f228e54615cac449:

2024-11-24T23:39:36,427 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=118.6 K, sizeToCheck=16.0 K
2024-11-24T23:39:36,427 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:36,427 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/fa04705436014aee90eed6c9a75036e6 because midkey is the same as first or last row
2024-11-24T23:39:36,433 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13f4128eec290306f228e54615cac449/info of 13f4128eec290306f228e54615cac449 into 96c490f97dd042c4bdefc1a1ff160658(size=87.4 K), total size for store is 109.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:36,433 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13f4128eec290306f228e54615cac449:

2024-11-24T23:39:36,434 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449., storeName=13f4128eec290306f228e54615cac449/info, priority=13, startTime=1732491576394; duration=0sec
2024-11-24T23:39:36,434 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=109.1 K, sizeToCheck=16.0 K
2024-11-24T23:39:36,434 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:36,434 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=109.1 K, sizeToCheck=16.0 K
2024-11-24T23:39:36,434 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:36,434 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=109.1 K, sizeToCheck=16.0 K
2024-11-24T23:39:36,434 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T23:39:36,436 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:36,436 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:36,436 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13f4128eec290306f228e54615cac449:info
2024-11-24T23:39:36,438 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35835 {}] assignment.AssignmentManager(1363): Split request from ba7f32532130,46359,1732491560986, parent={ENCODED => 13f4128eec290306f228e54615cac449, NAME => 'TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-24T23:39:36,445 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35835 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=ba7f32532130,46359,1732491560986
2024-11-24T23:39:36,451 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35835 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=13f4128eec290306f228e54615cac449, daughterA=2663dd450f9f98026c70e9ee11bb8094, daughterB=1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:36,453 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=13f4128eec290306f228e54615cac449, daughterA=2663dd450f9f98026c70e9ee11bb8094, daughterB=1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:36,453 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=13f4128eec290306f228e54615cac449, daughterA=2663dd450f9f98026c70e9ee11bb8094, daughterB=1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:36,453 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=13f4128eec290306f228e54615cac449, daughterA=2663dd450f9f98026c70e9ee11bb8094, daughterB=1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:36,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:36,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:36,549 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=13f4128eec290306f228e54615cac449, UNASSIGN}]
2024-11-24T23:39:36,550 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=13f4128eec290306f228e54615cac449, UNASSIGN
2024-11-24T23:39:36,552 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=13f4128eec290306f228e54615cac449, regionState=CLOSING, regionLocation=ba7f32532130,46359,1732491560986
2024-11-24T23:39:36,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=13f4128eec290306f228e54615cac449, UNASSIGN because future has completed
2024-11-24T23:39:36,555 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-24T23:39:36,555 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13f4128eec290306f228e54615cac449, server=ba7f32532130,46359,1732491560986}]
2024-11-24T23:39:36,713 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,713 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-24T23:39:36,714 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 13f4128eec290306f228e54615cac449, disabling compactions & flushes
2024-11-24T23:39:36,714 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:36,714 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:36,714 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449. after waiting 0 ms
2024-11-24T23:39:36,714 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:36,714 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 13f4128eec290306f228e54615cac449 1/1 column families, dataSize=2.10 KB heapSize=2.50 KB
2024-11-24T23:39:36,719 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/b78366cee3a742a8b0b89625a1bb3647 is 1080, key is row0095/info:/1732491576397/Put/seqid=0
2024-11-24T23:39:36,730 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741848_1024 (size=7112)
2024-11-24T23:39:36,730 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741848_1024 (size=7112)
2024-11-24T23:39:36,733 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.10 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/b78366cee3a742a8b0b89625a1bb3647
2024-11-24T23:39:36,740 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/.tmp/info/b78366cee3a742a8b0b89625a1bb3647 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/b78366cee3a742a8b0b89625a1bb3647
2024-11-24T23:39:36,747 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/b78366cee3a742a8b0b89625a1bb3647, entries=2, sequenceid=127, filesize=6.9 K
2024-11-24T23:39:36,748 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~2.10 KB/2152, heapSize ~2.48 KB/2544, currentSize=0 B/0 for 13f4128eec290306f228e54615cac449 in 34ms, sequenceid=127, compaction requested=true
2024-11-24T23:39:36,749 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/e6992a9ba01b4f61aaa23420b9fb88a9, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/379d098d924f415689da01db8b119a44, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/3c6475a2ed5e4d9ea826f59a3bb18536, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/589ac39d09484770b578c2ec9fcbfe26, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/b2c04e83db9445c0ba874d5ada27430b, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/fa04705436014aee90eed6c9a75036e6, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/72ed511a7a3240bb993eda41d62af8c6, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/ef103b296db74fb88cfd45d1918d0aef, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/369ecb3b735b46088670bf9be95ba99f] to archive
2024-11-24T23:39:36,751 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T23:39:36,754 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/e6992a9ba01b4f61aaa23420b9fb88a9 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/e6992a9ba01b4f61aaa23420b9fb88a9
2024-11-24T23:39:36,755 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/379d098d924f415689da01db8b119a44 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/379d098d924f415689da01db8b119a44
2024-11-24T23:39:36,757 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/3c6475a2ed5e4d9ea826f59a3bb18536 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/3c6475a2ed5e4d9ea826f59a3bb18536
2024-11-24T23:39:36,758 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/589ac39d09484770b578c2ec9fcbfe26 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/589ac39d09484770b578c2ec9fcbfe26
2024-11-24T23:39:36,759 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/b2c04e83db9445c0ba874d5ada27430b to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/b2c04e83db9445c0ba874d5ada27430b
2024-11-24T23:39:36,760 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/fa04705436014aee90eed6c9a75036e6 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/fa04705436014aee90eed6c9a75036e6
2024-11-24T23:39:36,762 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/72ed511a7a3240bb993eda41d62af8c6 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/72ed511a7a3240bb993eda41d62af8c6
2024-11-24T23:39:36,763 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/ef103b296db74fb88cfd45d1918d0aef to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/ef103b296db74fb88cfd45d1918d0aef
2024-11-24T23:39:36,764 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/369ecb3b735b46088670bf9be95ba99f to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/369ecb3b735b46088670bf9be95ba99f
2024-11-24T23:39:36,770 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-11-24T23:39:36,771 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.
2024-11-24T23:39:36,771 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 13f4128eec290306f228e54615cac449:
Waiting for close lock at 1732491576714Running coprocessor pre-close hooks at 1732491576714Disabling compacts and flushes for region at 1732491576714Disabling writes for close at 1732491576714Obtaining lock to block concurrent updates at 1732491576714Preparing flush snapshotting stores in 13f4128eec290306f228e54615cac449 at 1732491576714Finished memstore snapshotting TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449., syncing WAL and waiting on mvcc, flushsize=dataSize=2152, getHeapSize=2544, getOffHeapSize=0, getCellsCount=2 at 1732491576715 (+1 ms)Flushing stores of TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449. at 1732491576716 (+1 ms)Flushing 13f4128eec290306f228e54615cac449/info: creating writer at 1732491576716Flushing 13f4128eec290306f228e54615cac449/info: appending metadata at 1732491576718 (+2 ms)Flushing 13f4128eec290306f228e54615cac449/info: closing flushed file at 1732491576719 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60b5c54a: reopening flushed file at 1732491576739 (+20 ms)Finished flush of dataSize ~2.10 KB/2152, heapSize ~2.48 KB/2544, currentSize=0 B/0 for 13f4128eec290306f228e54615cac449 in 34ms, sequenceid=127, compaction requested=true at 1732491576748 (+9 ms)Writing region close event to WAL at 1732491576767 (+19 ms)Running coprocessor post-close hooks at 1732491576771 (+4 ms)Closed at 1732491576771
2024-11-24T23:39:36,773 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,774 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=13f4128eec290306f228e54615cac449, regionState=CLOSED
2024-11-24T23:39:36,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13f4128eec290306f228e54615cac449, server=ba7f32532130,46359,1732491560986 because future has completed
2024-11-24T23:39:36,779 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-24T23:39:36,779 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 13f4128eec290306f228e54615cac449, server=ba7f32532130,46359,1732491560986 in 222 msec
2024-11-24T23:39:36,783 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-24T23:39:36,783 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=13f4128eec290306f228e54615cac449, UNASSIGN in 230 msec
2024-11-24T23:39:36,793 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:36,796 INFO  [PEWorker-3 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=13f4128eec290306f228e54615cac449, threads=3
2024-11-24T23:39:36,798 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/05ad70c1cf6b44e38bdeeab50a87ceeb for region: 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,798 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/96c490f97dd042c4bdefc1a1ff160658 for region: 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,800 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/b78366cee3a742a8b0b89625a1bb3647 for region: 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,809 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/b78366cee3a742a8b0b89625a1bb3647, top=true
2024-11-24T23:39:36,809 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/05ad70c1cf6b44e38bdeeab50a87ceeb, top=true
2024-11-24T23:39:36,815 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-05ad70c1cf6b44e38bdeeab50a87ceeb for child: 1052af1b21d3f2f69648c2a80dedaae0, parent: 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,816 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741849_1025 (size=27)
2024-11-24T23:39:36,816 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/05ad70c1cf6b44e38bdeeab50a87ceeb for region: 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,817 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741849_1025 (size=27)
2024-11-24T23:39:36,817 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-b78366cee3a742a8b0b89625a1bb3647 for child: 1052af1b21d3f2f69648c2a80dedaae0, parent: 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,817 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/b78366cee3a742a8b0b89625a1bb3647 for region: 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741850_1026 (size=27)
2024-11-24T23:39:36,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741850_1026 (size=27)
2024-11-24T23:39:36,829 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/96c490f97dd042c4bdefc1a1ff160658 for region: 13f4128eec290306f228e54615cac449
2024-11-24T23:39:36,831 DEBUG [PEWorker-3 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 13f4128eec290306f228e54615cac449 Daughter A: [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449] storefiles, Daughter B: [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-05ad70c1cf6b44e38bdeeab50a87ceeb, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-b78366cee3a742a8b0b89625a1bb3647] storefiles.
2024-11-24T23:39:36,864 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741851_1027 (size=71)
2024-11-24T23:39:36,864 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741851_1027 (size=71)
2024-11-24T23:39:36,870 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:36,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741852_1028 (size=71)
2024-11-24T23:39:36,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741852_1028 (size=71)
2024-11-24T23:39:36,892 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:36,906 DEBUG [PEWorker-3 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-24T23:39:36,909 DEBUG [PEWorker-3 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-24T23:39:36,912 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732491576911"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1732491576911"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1732491576911"}]},"ts":"1732491576911"}
2024-11-24T23:39:36,912 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732491576911"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732491576911"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732491576911"}]},"ts":"1732491576911"}
2024-11-24T23:39:36,912 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732491576911"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732491576911"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732491576911"}]},"ts":"1732491576911"}
2024-11-24T23:39:36,932 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=2663dd450f9f98026c70e9ee11bb8094, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1052af1b21d3f2f69648c2a80dedaae0, ASSIGN}]
2024-11-24T23:39:36,933 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=2663dd450f9f98026c70e9ee11bb8094, ASSIGN
2024-11-24T23:39:36,933 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1052af1b21d3f2f69648c2a80dedaae0, ASSIGN
2024-11-24T23:39:36,934 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=2663dd450f9f98026c70e9ee11bb8094, ASSIGN; state=SPLITTING_NEW, location=ba7f32532130,46359,1732491560986; forceNewPlan=false, retain=false
2024-11-24T23:39:36,934 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1052af1b21d3f2f69648c2a80dedaae0, ASSIGN; state=SPLITTING_NEW, location=ba7f32532130,46359,1732491560986; forceNewPlan=false, retain=false
2024-11-24T23:39:37,084 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1052af1b21d3f2f69648c2a80dedaae0, regionState=OPENING, regionLocation=ba7f32532130,46359,1732491560986
2024-11-24T23:39:37,084 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=2663dd450f9f98026c70e9ee11bb8094, regionState=OPENING, regionLocation=ba7f32532130,46359,1732491560986
2024-11-24T23:39:37,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1052af1b21d3f2f69648c2a80dedaae0, ASSIGN because future has completed
2024-11-24T23:39:37,088 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1052af1b21d3f2f69648c2a80dedaae0, server=ba7f32532130,46359,1732491560986}]
2024-11-24T23:39:37,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=2663dd450f9f98026c70e9ee11bb8094, ASSIGN because future has completed
2024-11-24T23:39:37,089 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2663dd450f9f98026c70e9ee11bb8094, server=ba7f32532130,46359,1732491560986}]
2024-11-24T23:39:37,245 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:37,245 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 1052af1b21d3f2f69648c2a80dedaae0, NAME => 'TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-24T23:39:37,245 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:37,245 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:39:37,245 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:37,245 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:37,247 INFO  [StoreOpener-1052af1b21d3f2f69648c2a80dedaae0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1052af1b21d3f2f69648c2a80dedaae0 
2024-11-24T23:39:37,247 INFO  [StoreOpener-1052af1b21d3f2f69648c2a80dedaae0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1052af1b21d3f2f69648c2a80dedaae0 columnFamilyName info
2024-11-24T23:39:37,247 DEBUG [StoreOpener-1052af1b21d3f2f69648c2a80dedaae0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:37,260 DEBUG [StoreOpener-1052af1b21d3f2f69648c2a80dedaae0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449->hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/96c490f97dd042c4bdefc1a1ff160658-top
2024-11-24T23:39:37,266 DEBUG [StoreOpener-1052af1b21d3f2f69648c2a80dedaae0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-05ad70c1cf6b44e38bdeeab50a87ceeb
2024-11-24T23:39:37,270 DEBUG [StoreOpener-1052af1b21d3f2f69648c2a80dedaae0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-b78366cee3a742a8b0b89625a1bb3647
2024-11-24T23:39:37,270 INFO  [StoreOpener-1052af1b21d3f2f69648c2a80dedaae0-1 {}] regionserver.HStore(327): Store=1052af1b21d3f2f69648c2a80dedaae0/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:39:37,271 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:37,272 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:37,273 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:37,273 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:37,273 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:37,275 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:37,276 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 1052af1b21d3f2f69648c2a80dedaae0; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=807995, jitterRate=0.027419954538345337}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T23:39:37,276 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:37,276 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 1052af1b21d3f2f69648c2a80dedaae0:
Running coprocessor pre-open hook at 1732491577246Writing region info on filesystem at 1732491577246Initializing all the Stores at 1732491577246Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491577246Cleaning up temporary data from old regions at 1732491577273 (+27 ms)Running coprocessor post-open hooks at 1732491577276 (+3 ms)Region opened successfully at 1732491577276
2024-11-24T23:39:37,277 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0., pid=12, masterSystemTime=1732491577241
2024-11-24T23:39:37,278 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 1052af1b21d3f2f69648c2a80dedaae0:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:39:37,278 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:37,278 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:37,280 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:37,280 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1541): 1052af1b21d3f2f69648c2a80dedaae0/info is initiating minor compaction (all files)
2024-11-24T23:39:37,280 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1052af1b21d3f2f69648c2a80dedaae0/info in TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:37,280 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449->hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/96c490f97dd042c4bdefc1a1ff160658-top, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-05ad70c1cf6b44e38bdeeab50a87ceeb, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-b78366cee3a742a8b0b89625a1bb3647] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp, totalSize=116.0 K
2024-11-24T23:39:37,280 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:37,280 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:37,280 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.
2024-11-24T23:39:37,280 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 2663dd450f9f98026c70e9ee11bb8094, NAME => 'TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-24T23:39:37,281 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449, keycount=39, bloomtype=ROW, size=87.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732491572184
2024-11-24T23:39:37,281 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:37,281 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1052af1b21d3f2f69648c2a80dedaae0, regionState=OPEN, openSeqNum=131, regionLocation=ba7f32532130,46359,1732491560986
2024-11-24T23:39:37,281 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:39:37,281 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:37,281 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:37,281 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-05ad70c1cf6b44e38bdeeab50a87ceeb, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732491576365
2024-11-24T23:39:37,281 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-b78366cee3a742a8b0b89625a1bb3647, keycount=2, bloomtype=ROW, size=6.9 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732491576397
2024-11-24T23:39:37,282 INFO  [StoreOpener-2663dd450f9f98026c70e9ee11bb8094-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 2663dd450f9f98026c70e9ee11bb8094 
2024-11-24T23:39:37,283 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-24T23:39:37,283 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-24T23:39:37,283 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB
2024-11-24T23:39:37,283 INFO  [StoreOpener-2663dd450f9f98026c70e9ee11bb8094-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2663dd450f9f98026c70e9ee11bb8094 columnFamilyName info
2024-11-24T23:39:37,283 DEBUG [StoreOpener-2663dd450f9f98026c70e9ee11bb8094-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:39:37,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1052af1b21d3f2f69648c2a80dedaae0, server=ba7f32532130,46359,1732491560986 because future has completed
2024-11-24T23:39:37,292 DEBUG [StoreOpener-2663dd450f9f98026c70e9ee11bb8094-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449->hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/96c490f97dd042c4bdefc1a1ff160658-bottom
2024-11-24T23:39:37,292 INFO  [StoreOpener-2663dd450f9f98026c70e9ee11bb8094-1 {}] regionserver.HStore(327): Store=2663dd450f9f98026c70e9ee11bb8094/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:39:37,292 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:37,293 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:37,294 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:37,295 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:37,295 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:37,298 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-24T23:39:37,298 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 1052af1b21d3f2f69648c2a80dedaae0, server=ba7f32532130,46359,1732491560986 in 197 msec
2024-11-24T23:39:37,298 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:37,300 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 2663dd450f9f98026c70e9ee11bb8094; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=752207, jitterRate=-0.043519869446754456}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T23:39:37,300 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:37,300 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 2663dd450f9f98026c70e9ee11bb8094:
Running coprocessor pre-open hook at 1732491577281Writing region info on filesystem at 1732491577281Initializing all the Stores at 1732491577282 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491577282Cleaning up temporary data from old regions at 1732491577295 (+13 ms)Running coprocessor post-open hooks at 1732491577300 (+5 ms)Region opened successfully at 1732491577300
2024-11-24T23:39:37,301 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1052af1b21d3f2f69648c2a80dedaae0, ASSIGN in 366 msec
2024-11-24T23:39:37,301 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094., pid=13, masterSystemTime=1732491577241
2024-11-24T23:39:37,302 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 2663dd450f9f98026c70e9ee11bb8094:info, priority=-2147483648, current under compaction store size is 2
2024-11-24T23:39:37,302 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:37,302 DEBUG [RS:0;ba7f32532130:46359-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-24T23:39:37,302 INFO  [RS:0;ba7f32532130:46359-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.
2024-11-24T23:39:37,302 DEBUG [RS:0;ba7f32532130:46359-longCompactions-0 {}] regionserver.HStore(1541): 2663dd450f9f98026c70e9ee11bb8094/info is initiating minor compaction (all files)
2024-11-24T23:39:37,302 INFO  [RS:0;ba7f32532130:46359-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2663dd450f9f98026c70e9ee11bb8094/info in TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.
2024-11-24T23:39:37,303 INFO  [RS:0;ba7f32532130:46359-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449->hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/96c490f97dd042c4bdefc1a1ff160658-bottom] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/.tmp, totalSize=87.4 K
2024-11-24T23:39:37,304 DEBUG [RS:0;ba7f32532130:46359-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449, keycount=39, bloomtype=ROW, size=87.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732491572184
2024-11-24T23:39:37,304 DEBUG [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.
2024-11-24T23:39:37,304 INFO  [RS_OPEN_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.
2024-11-24T23:39:37,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/info/60c8f8b6ed6b40028ff9ec80a5475738 is 193, key is TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0./info:regioninfo/1732491577281/Put/seqid=0
2024-11-24T23:39:37,306 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=2663dd450f9f98026c70e9ee11bb8094, regionState=OPEN, openSeqNum=131, regionLocation=ba7f32532130,46359,1732491560986
2024-11-24T23:39:37,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2663dd450f9f98026c70e9ee11bb8094, server=ba7f32532130,46359,1732491560986 because future has completed
2024-11-24T23:39:37,311 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10
2024-11-24T23:39:37,311 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 2663dd450f9f98026c70e9ee11bb8094, server=ba7f32532130,46359,1732491560986 in 220 msec
2024-11-24T23:39:37,314 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7
2024-11-24T23:39:37,315 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=2663dd450f9f98026c70e9ee11bb8094, ASSIGN in 379 msec
2024-11-24T23:39:37,315 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=13f4128eec290306f228e54615cac449, daughterA=2663dd450f9f98026c70e9ee11bb8094, daughterB=1052af1b21d3f2f69648c2a80dedaae0 in 869 msec
2024-11-24T23:39:37,317 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1052af1b21d3f2f69648c2a80dedaae0#info#compaction#68 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:37,317 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/b4937c41e81f413ca12ef11c808d7b48 is 1080, key is row0062/info:/1732491574317/Put/seqid=0
2024-11-24T23:39:37,329 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741853_1029 (size=9882)
2024-11-24T23:39:37,329 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741853_1029 (size=9882)
2024-11-24T23:39:37,330 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/info/60c8f8b6ed6b40028ff9ec80a5475738
2024-11-24T23:39:37,331 INFO  [RS:0;ba7f32532130:46359-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2663dd450f9f98026c70e9ee11bb8094#info#compaction#69 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:37,332 DEBUG [RS:0;ba7f32532130:46359-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/.tmp/info/477eb54a13d844d4b934bec63755960f is 1080, key is row0001/info:/1732491572184/Put/seqid=0
2024-11-24T23:39:37,341 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741854_1030 (size=42984)
2024-11-24T23:39:37,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741854_1030 (size=42984)
2024-11-24T23:39:37,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741855_1031 (size=70862)
2024-11-24T23:39:37,343 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741855_1031 (size=70862)
2024-11-24T23:39:37,348 DEBUG [RS:0;ba7f32532130:46359-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/.tmp/info/477eb54a13d844d4b934bec63755960f as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/info/477eb54a13d844d4b934bec63755960f
2024-11-24T23:39:37,353 INFO  [RS:0;ba7f32532130:46359-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 2663dd450f9f98026c70e9ee11bb8094/info of 2663dd450f9f98026c70e9ee11bb8094 into 477eb54a13d844d4b934bec63755960f(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:37,353 DEBUG [RS:0;ba7f32532130:46359-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2663dd450f9f98026c70e9ee11bb8094:

2024-11-24T23:39:37,353 INFO  [RS:0;ba7f32532130:46359-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094., storeName=2663dd450f9f98026c70e9ee11bb8094/info, priority=15, startTime=1732491577301; duration=0sec
2024-11-24T23:39:37,353 DEBUG [RS:0;ba7f32532130:46359-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:37,353 DEBUG [RS:0;ba7f32532130:46359-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2663dd450f9f98026c70e9ee11bb8094:info
2024-11-24T23:39:37,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/ns/8a7fa2ff584d4aec94f1b57cd122d2db is 43, key is default/ns:d/1732491562041/Put/seqid=0
2024-11-24T23:39:37,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741856_1032 (size=5153)
2024-11-24T23:39:37,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741856_1032 (size=5153)
2024-11-24T23:39:37,358 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/ns/8a7fa2ff584d4aec94f1b57cd122d2db
2024-11-24T23:39:37,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/table/990b8b5e8c844cfca602ecaefe6081db is 65, key is TestLogRolling-testLogRolling/table:state/1732491562499/Put/seqid=0
2024-11-24T23:39:37,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741857_1033 (size=5340)
2024-11-24T23:39:37,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741857_1033 (size=5340)
2024-11-24T23:39:37,382 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/table/990b8b5e8c844cfca602ecaefe6081db
2024-11-24T23:39:37,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/info/60c8f8b6ed6b40028ff9ec80a5475738 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/info/60c8f8b6ed6b40028ff9ec80a5475738
2024-11-24T23:39:37,392 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/info/60c8f8b6ed6b40028ff9ec80a5475738, entries=30, sequenceid=17, filesize=9.7 K
2024-11-24T23:39:37,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/ns/8a7fa2ff584d4aec94f1b57cd122d2db as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/ns/8a7fa2ff584d4aec94f1b57cd122d2db
2024-11-24T23:39:37,399 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/ns/8a7fa2ff584d4aec94f1b57cd122d2db, entries=2, sequenceid=17, filesize=5.0 K
2024-11-24T23:39:37,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/table/990b8b5e8c844cfca602ecaefe6081db as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/table/990b8b5e8c844cfca602ecaefe6081db
2024-11-24T23:39:37,404 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/table/990b8b5e8c844cfca602ecaefe6081db, entries=2, sequenceid=17, filesize=5.2 K
2024-11-24T23:39:37,405 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 122ms, sequenceid=17, compaction requested=false
2024-11-24T23:39:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-24T23:39:37,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:37,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:37,757 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/b4937c41e81f413ca12ef11c808d7b48 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/b4937c41e81f413ca12ef11c808d7b48
2024-11-24T23:39:37,766 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1052af1b21d3f2f69648c2a80dedaae0/info of 1052af1b21d3f2f69648c2a80dedaae0 into b4937c41e81f413ca12ef11c808d7b48(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:37,766 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:37,766 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0., storeName=1052af1b21d3f2f69648c2a80dedaae0/info, priority=13, startTime=1732491577277; duration=0sec
2024-11-24T23:39:37,766 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:37,766 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1052af1b21d3f2f69648c2a80dedaae0:info
2024-11-24T23:39:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:57516 deadline: 1732491588405, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449. is not online on ba7f32532130,46359,1732491560986
2024-11-24T23:39:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449., hostname=ba7f32532130,46359,1732491560986, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449., hostname=ba7f32532130,46359,1732491560986, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449. is not online on ba7f32532130,46359,1732491560986
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-24T23:39:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449., hostname=ba7f32532130,46359,1732491560986, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449. is not online on ba7f32532130,46359,1732491560986
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-24T23:39:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1732491562125.13f4128eec290306f228e54615cac449., hostname=ba7f32532130,46359,1732491560986, seqNum=2 from cache
2024-11-24T23:39:38,488 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:38,544 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:39,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:39,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:40,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:40,546 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:41,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:41,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:41,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,773 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,774 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,774 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,775 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,775 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,777 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,777 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,800 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,801 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,804 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,804 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,804 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:41,806 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,313 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T23:39:42,316 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,316 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,316 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,318 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,319 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,341 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,346 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,346 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,346 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,348 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:39:42,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:42,549 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:43,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:43,550 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:44,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:44,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:45,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:45,553 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:46,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:46,554 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:47,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:47,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:48,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0., hostname=ba7f32532130,46359,1732491560986, seqNum=131]
2024-11-24T23:39:48,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:48,569 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T23:39:48,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/6ebff42cb0ef4791ad78ef9c6d7f9373 is 1080, key is row0097/info:/1732491588556/Put/seqid=0
2024-11-24T23:39:48,578 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741858_1034 (size=12516)
2024-11-24T23:39:48,579 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741858_1034 (size=12516)
2024-11-24T23:39:48,579 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/6ebff42cb0ef4791ad78ef9c6d7f9373
2024-11-24T23:39:48,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/6ebff42cb0ef4791ad78ef9c6d7f9373 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6ebff42cb0ef4791ad78ef9c6d7f9373
2024-11-24T23:39:48,593 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6ebff42cb0ef4791ad78ef9c6d7f9373, entries=7, sequenceid=141, filesize=12.2 K
2024-11-24T23:39:48,594 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 1052af1b21d3f2f69648c2a80dedaae0 in 24ms, sequenceid=141, compaction requested=false
2024-11-24T23:39:48,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:48,594 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-24T23:39:48,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/3812fd69754944438c95cae42aeebd8e is 1080, key is row0104/info:/1732491588569/Put/seqid=0
2024-11-24T23:39:48,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741859_1035 (size=19000)
2024-11-24T23:39:48,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741859_1035 (size=19000)
2024-11-24T23:39:48,604 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/3812fd69754944438c95cae42aeebd8e
2024-11-24T23:39:48,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/3812fd69754944438c95cae42aeebd8e as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3812fd69754944438c95cae42aeebd8e
2024-11-24T23:39:48,616 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3812fd69754944438c95cae42aeebd8e, entries=13, sequenceid=157, filesize=18.6 K
2024-11-24T23:39:48,617 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=12.61 KB/12912 for 1052af1b21d3f2f69648c2a80dedaae0 in 23ms, sequenceid=157, compaction requested=true
2024-11-24T23:39:48,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:48,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1052af1b21d3f2f69648c2a80dedaae0:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:39:48,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:48,617 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:48,618 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 74500 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:39:48,618 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1541): 1052af1b21d3f2f69648c2a80dedaae0/info is initiating minor compaction (all files)
2024-11-24T23:39:48,618 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1052af1b21d3f2f69648c2a80dedaae0/info in TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:48,619 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/b4937c41e81f413ca12ef11c808d7b48, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6ebff42cb0ef4791ad78ef9c6d7f9373, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3812fd69754944438c95cae42aeebd8e] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp, totalSize=72.8 K
2024-11-24T23:39:48,619 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4937c41e81f413ca12ef11c808d7b48, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732491574317
2024-11-24T23:39:48,619 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ebff42cb0ef4791ad78ef9c6d7f9373, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732491588556
2024-11-24T23:39:48,620 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3812fd69754944438c95cae42aeebd8e, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732491588569
2024-11-24T23:39:48,630 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1052af1b21d3f2f69648c2a80dedaae0#info#compaction#74 average throughput is 56.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:48,630 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/d885063f9c584259930e1f3320a5d9a4 is 1080, key is row0062/info:/1732491574317/Put/seqid=0
2024-11-24T23:39:48,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741860_1036 (size=64714)
2024-11-24T23:39:48,635 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741860_1036 (size=64714)
2024-11-24T23:39:48,641 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/d885063f9c584259930e1f3320a5d9a4 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/d885063f9c584259930e1f3320a5d9a4
2024-11-24T23:39:48,648 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1052af1b21d3f2f69648c2a80dedaae0/info of 1052af1b21d3f2f69648c2a80dedaae0 into d885063f9c584259930e1f3320a5d9a4(size=63.2 K), total size for store is 63.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:48,648 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:48,648 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0., storeName=1052af1b21d3f2f69648c2a80dedaae0/info, priority=13, startTime=1732491588617; duration=0sec
2024-11-24T23:39:48,648 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:48,648 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1052af1b21d3f2f69648c2a80dedaae0:info
2024-11-24T23:39:49,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:49,557 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:50,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:50,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:50,623 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-24T23:39:50,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/a54033f2f8b74ea68799e26ed865cd82 is 1080, key is row0117/info:/1732491588595/Put/seqid=0
2024-11-24T23:39:50,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741861_1037 (size=19000)
2024-11-24T23:39:50,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741861_1037 (size=19000)
2024-11-24T23:39:50,640 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/a54033f2f8b74ea68799e26ed865cd82
2024-11-24T23:39:50,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/a54033f2f8b74ea68799e26ed865cd82 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a54033f2f8b74ea68799e26ed865cd82
2024-11-24T23:39:50,652 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a54033f2f8b74ea68799e26ed865cd82, entries=13, sequenceid=174, filesize=18.6 K
2024-11-24T23:39:50,653 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=11.56 KB/11836 for 1052af1b21d3f2f69648c2a80dedaae0 in 29ms, sequenceid=174, compaction requested=false
2024-11-24T23:39:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:50,654 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-24T23:39:50,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/10187fa2595149cdbdfc7e4668739399 is 1080, key is row0130/info:/1732491590626/Put/seqid=0
2024-11-24T23:39:50,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741862_1038 (size=17906)
2024-11-24T23:39:50,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741862_1038 (size=17906)
2024-11-24T23:39:50,663 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/10187fa2595149cdbdfc7e4668739399
2024-11-24T23:39:50,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/10187fa2595149cdbdfc7e4668739399 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/10187fa2595149cdbdfc7e4668739399
2024-11-24T23:39:50,674 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/10187fa2595149cdbdfc7e4668739399, entries=12, sequenceid=189, filesize=17.5 K
2024-11-24T23:39:50,675 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 1052af1b21d3f2f69648c2a80dedaae0 in 21ms, sequenceid=189, compaction requested=true
2024-11-24T23:39:50,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:50,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1052af1b21d3f2f69648c2a80dedaae0:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:39:50,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:50,675 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:50,676 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-24T23:39:50,677 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 101620 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:39:50,677 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1541): 1052af1b21d3f2f69648c2a80dedaae0/info is initiating minor compaction (all files)
2024-11-24T23:39:50,677 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1052af1b21d3f2f69648c2a80dedaae0/info in TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:50,677 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/d885063f9c584259930e1f3320a5d9a4, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a54033f2f8b74ea68799e26ed865cd82, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/10187fa2595149cdbdfc7e4668739399] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp, totalSize=99.2 K
2024-11-24T23:39:50,677 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting d885063f9c584259930e1f3320a5d9a4, keycount=55, bloomtype=ROW, size=63.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732491574317
2024-11-24T23:39:50,678 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting a54033f2f8b74ea68799e26ed865cd82, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732491588595
2024-11-24T23:39:50,678 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10187fa2595149cdbdfc7e4668739399, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732491590626
2024-11-24T23:39:50,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/35f5e504d761407b993be2aec36e4b87 is 1080, key is row0142/info:/1732491590655/Put/seqid=0
2024-11-24T23:39:50,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741863_1039 (size=16828)
2024-11-24T23:39:50,686 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741863_1039 (size=16828)
2024-11-24T23:39:50,686 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/35f5e504d761407b993be2aec36e4b87
2024-11-24T23:39:50,692 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1052af1b21d3f2f69648c2a80dedaae0#info#compaction#78 average throughput is 41.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:50,692 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/f641e7f016d9405381c83883ebe7d7a7 is 1080, key is row0062/info:/1732491574317/Put/seqid=0
2024-11-24T23:39:50,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/35f5e504d761407b993be2aec36e4b87 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/35f5e504d761407b993be2aec36e4b87
2024-11-24T23:39:50,699 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741864_1040 (size=91843)
2024-11-24T23:39:50,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741864_1040 (size=91843)
2024-11-24T23:39:50,702 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/35f5e504d761407b993be2aec36e4b87, entries=11, sequenceid=203, filesize=16.4 K
2024-11-24T23:39:50,703 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=8.41 KB/8608 for 1052af1b21d3f2f69648c2a80dedaae0 in 27ms, sequenceid=203, compaction requested=false
2024-11-24T23:39:50,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:50,705 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/f641e7f016d9405381c83883ebe7d7a7 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/f641e7f016d9405381c83883ebe7d7a7
2024-11-24T23:39:50,711 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1052af1b21d3f2f69648c2a80dedaae0/info of 1052af1b21d3f2f69648c2a80dedaae0 into f641e7f016d9405381c83883ebe7d7a7(size=89.7 K), total size for store is 106.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:50,711 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:50,711 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0., storeName=1052af1b21d3f2f69648c2a80dedaae0/info, priority=13, startTime=1732491590675; duration=0sec
2024-11-24T23:39:50,711 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:50,711 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1052af1b21d3f2f69648c2a80dedaae0:info
2024-11-24T23:39:50,768 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T23:39:51,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:51,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:52,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:52,561 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:52,700 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-24T23:39:52,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/165e6431b5a74ff588076317d5d2d4cf is 1080, key is row0153/info:/1732491590678/Put/seqid=0
2024-11-24T23:39:52,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741865_1041 (size=14672)
2024-11-24T23:39:52,714 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741865_1041 (size=14672)
2024-11-24T23:39:52,714 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/165e6431b5a74ff588076317d5d2d4cf
2024-11-24T23:39:52,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/165e6431b5a74ff588076317d5d2d4cf as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/165e6431b5a74ff588076317d5d2d4cf
2024-11-24T23:39:52,727 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/165e6431b5a74ff588076317d5d2d4cf, entries=9, sequenceid=216, filesize=14.3 K
2024-11-24T23:39:52,728 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=11.56 KB/11836 for 1052af1b21d3f2f69648c2a80dedaae0 in 29ms, sequenceid=216, compaction requested=true
2024-11-24T23:39:52,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:52,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1052af1b21d3f2f69648c2a80dedaae0:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:39:52,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:52,728 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:52,729 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-24T23:39:52,729 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 123343 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:39:52,729 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1541): 1052af1b21d3f2f69648c2a80dedaae0/info is initiating minor compaction (all files)
2024-11-24T23:39:52,730 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1052af1b21d3f2f69648c2a80dedaae0/info in TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:52,730 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/f641e7f016d9405381c83883ebe7d7a7, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/35f5e504d761407b993be2aec36e4b87, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/165e6431b5a74ff588076317d5d2d4cf] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp, totalSize=120.5 K
2024-11-24T23:39:52,730 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting f641e7f016d9405381c83883ebe7d7a7, keycount=80, bloomtype=ROW, size=89.7 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732491574317
2024-11-24T23:39:52,731 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35f5e504d761407b993be2aec36e4b87, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732491590655
2024-11-24T23:39:52,731 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 165e6431b5a74ff588076317d5d2d4cf, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732491590678
2024-11-24T23:39:52,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/5a89f465a35042ae8cb9ad3a20d3293c is 1080, key is row0162/info:/1732491592703/Put/seqid=0
2024-11-24T23:39:52,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741866_1042 (size=17906)
2024-11-24T23:39:52,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741866_1042 (size=17906)
2024-11-24T23:39:52,739 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/5a89f465a35042ae8cb9ad3a20d3293c
2024-11-24T23:39:52,744 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1052af1b21d3f2f69648c2a80dedaae0#info#compaction#81 average throughput is 34.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:52,744 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/1c3a3880d9394726b1c92ce255c2c1af is 1080, key is row0062/info:/1732491574317/Put/seqid=0
2024-11-24T23:39:52,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/5a89f465a35042ae8cb9ad3a20d3293c as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/5a89f465a35042ae8cb9ad3a20d3293c
2024-11-24T23:39:52,748 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741867_1043 (size=113509)
2024-11-24T23:39:52,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741867_1043 (size=113509)
2024-11-24T23:39:52,752 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/5a89f465a35042ae8cb9ad3a20d3293c, entries=12, sequenceid=231, filesize=17.5 K
2024-11-24T23:39:52,753 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 1052af1b21d3f2f69648c2a80dedaae0 in 24ms, sequenceid=231, compaction requested=false
2024-11-24T23:39:52,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:52,754 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-24T23:39:52,755 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/1c3a3880d9394726b1c92ce255c2c1af as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/1c3a3880d9394726b1c92ce255c2c1af
2024-11-24T23:39:52,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/e6d908c60cf246c9ab318063fff20da3 is 1080, key is row0174/info:/1732491592730/Put/seqid=0
2024-11-24T23:39:52,762 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1052af1b21d3f2f69648c2a80dedaae0/info of 1052af1b21d3f2f69648c2a80dedaae0 into 1c3a3880d9394726b1c92ce255c2c1af(size=110.8 K), total size for store is 128.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:52,762 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:52,762 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0., storeName=1052af1b21d3f2f69648c2a80dedaae0/info, priority=13, startTime=1732491592728; duration=0sec
2024-11-24T23:39:52,762 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:52,762 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1052af1b21d3f2f69648c2a80dedaae0:info
2024-11-24T23:39:52,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741868_1044 (size=16828)
2024-11-24T23:39:52,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741868_1044 (size=16828)
2024-11-24T23:39:52,769 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/e6d908c60cf246c9ab318063fff20da3
2024-11-24T23:39:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/e6d908c60cf246c9ab318063fff20da3 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e6d908c60cf246c9ab318063fff20da3
2024-11-24T23:39:52,778 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e6d908c60cf246c9ab318063fff20da3, entries=11, sequenceid=245, filesize=16.4 K
2024-11-24T23:39:52,779 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=8.41 KB/8608 for 1052af1b21d3f2f69648c2a80dedaae0 in 25ms, sequenceid=245, compaction requested=true
2024-11-24T23:39:52,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:52,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1052af1b21d3f2f69648c2a80dedaae0:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:39:52,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:52,779 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:52,780 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 148243 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:39:52,781 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1541): 1052af1b21d3f2f69648c2a80dedaae0/info is initiating minor compaction (all files)
2024-11-24T23:39:52,781 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1052af1b21d3f2f69648c2a80dedaae0/info in TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:52,781 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/1c3a3880d9394726b1c92ce255c2c1af, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/5a89f465a35042ae8cb9ad3a20d3293c, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e6d908c60cf246c9ab318063fff20da3] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp, totalSize=144.8 K
2024-11-24T23:39:52,781 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c3a3880d9394726b1c92ce255c2c1af, keycount=100, bloomtype=ROW, size=110.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732491574317
2024-11-24T23:39:52,781 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a89f465a35042ae8cb9ad3a20d3293c, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732491592703
2024-11-24T23:39:52,782 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6d908c60cf246c9ab318063fff20da3, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732491592730
2024-11-24T23:39:52,792 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1052af1b21d3f2f69648c2a80dedaae0#info#compaction#83 average throughput is 42.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:52,793 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/3489b5c8c92e4f74bf2df981ee264688 is 1080, key is row0062/info:/1732491574317/Put/seqid=0
2024-11-24T23:39:52,795 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741869_1045 (size=138610)
2024-11-24T23:39:52,795 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741869_1045 (size=138610)
2024-11-24T23:39:52,800 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/3489b5c8c92e4f74bf2df981ee264688 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3489b5c8c92e4f74bf2df981ee264688
2024-11-24T23:39:52,805 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1052af1b21d3f2f69648c2a80dedaae0/info of 1052af1b21d3f2f69648c2a80dedaae0 into 3489b5c8c92e4f74bf2df981ee264688(size=135.4 K), total size for store is 135.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:52,805 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:52,805 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0., storeName=1052af1b21d3f2f69648c2a80dedaae0/info, priority=13, startTime=1732491592779; duration=0sec
2024-11-24T23:39:52,805 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:52,805 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1052af1b21d3f2f69648c2a80dedaae0:info
2024-11-24T23:39:53,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:53,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:54,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:54,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:54,775 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-24T23:39:54,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/e694987a15304f7fa135215efa732662 is 1080, key is row0185/info:/1732491592755/Put/seqid=0
2024-11-24T23:39:54,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741870_1046 (size=14673)
2024-11-24T23:39:54,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741870_1046 (size=14673)
2024-11-24T23:39:54,791 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/e694987a15304f7fa135215efa732662
2024-11-24T23:39:54,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/e694987a15304f7fa135215efa732662 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e694987a15304f7fa135215efa732662
2024-11-24T23:39:54,802 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e694987a15304f7fa135215efa732662, entries=9, sequenceid=259, filesize=14.3 K
2024-11-24T23:39:54,802 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=9.46 KB/9684 for 1052af1b21d3f2f69648c2a80dedaae0 in 27ms, sequenceid=259, compaction requested=false
2024-11-24T23:39:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:54,805 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-24T23:39:54,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/74b316d5e07b4423893ea81ae908983e is 1080, key is row0194/info:/1732491594778/Put/seqid=0
2024-11-24T23:39:54,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741871_1047 (size=16839)
2024-11-24T23:39:54,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741871_1047 (size=16839)
2024-11-24T23:39:54,814 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/74b316d5e07b4423893ea81ae908983e
2024-11-24T23:39:54,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/74b316d5e07b4423893ea81ae908983e as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/74b316d5e07b4423893ea81ae908983e
2024-11-24T23:39:54,825 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/74b316d5e07b4423893ea81ae908983e, entries=11, sequenceid=273, filesize=16.4 K
2024-11-24T23:39:54,826 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for 1052af1b21d3f2f69648c2a80dedaae0 in 22ms, sequenceid=273, compaction requested=true
2024-11-24T23:39:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1052af1b21d3f2f69648c2a80dedaae0:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:39:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:54,826 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:54,827 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-24T23:39:54,827 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 170122 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:39:54,827 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1541): 1052af1b21d3f2f69648c2a80dedaae0/info is initiating minor compaction (all files)
2024-11-24T23:39:54,827 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1052af1b21d3f2f69648c2a80dedaae0/info in TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:54,827 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3489b5c8c92e4f74bf2df981ee264688, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e694987a15304f7fa135215efa732662, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/74b316d5e07b4423893ea81ae908983e] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp, totalSize=166.1 K
2024-11-24T23:39:54,828 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3489b5c8c92e4f74bf2df981ee264688, keycount=123, bloomtype=ROW, size=135.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732491574317
2024-11-24T23:39:54,828 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting e694987a15304f7fa135215efa732662, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732491592755
2024-11-24T23:39:54,828 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74b316d5e07b4423893ea81ae908983e, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732491594778
2024-11-24T23:39:54,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/f765a28969364505adc9d3bb29b688a4 is 1080, key is row0205/info:/1732491594806/Put/seqid=0
2024-11-24T23:39:54,835 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741872_1048 (size=17918)
2024-11-24T23:39:54,835 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741872_1048 (size=17918)
2024-11-24T23:39:54,836 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/f765a28969364505adc9d3bb29b688a4
2024-11-24T23:39:54,841 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1052af1b21d3f2f69648c2a80dedaae0#info#compaction#87 average throughput is 48.91 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:54,841 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/9f2d8f2c4c5f4dbe9ceb45b3074d2d0d is 1080, key is row0062/info:/1732491574317/Put/seqid=0
2024-11-24T23:39:54,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/f765a28969364505adc9d3bb29b688a4 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/f765a28969364505adc9d3bb29b688a4
2024-11-24T23:39:54,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741873_1049 (size=160288)
2024-11-24T23:39:54,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741873_1049 (size=160288)
2024-11-24T23:39:54,852 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/f765a28969364505adc9d3bb29b688a4, entries=12, sequenceid=288, filesize=17.5 K
2024-11-24T23:39:54,852 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/9f2d8f2c4c5f4dbe9ceb45b3074d2d0d as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/9f2d8f2c4c5f4dbe9ceb45b3074d2d0d
2024-11-24T23:39:54,853 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=8.41 KB/8608 for 1052af1b21d3f2f69648c2a80dedaae0 in 26ms, sequenceid=288, compaction requested=false
2024-11-24T23:39:54,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:54,857 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1052af1b21d3f2f69648c2a80dedaae0/info of 1052af1b21d3f2f69648c2a80dedaae0 into 9f2d8f2c4c5f4dbe9ceb45b3074d2d0d(size=156.5 K), total size for store is 174.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:54,858 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:54,858 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0., storeName=1052af1b21d3f2f69648c2a80dedaae0/info, priority=13, startTime=1732491594826; duration=0sec
2024-11-24T23:39:54,858 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:54,858 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1052af1b21d3f2f69648c2a80dedaae0:info
2024-11-24T23:39:55,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:55,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:56,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:56,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:56,849 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-24T23:39:56,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/51916c253f8145798a9843d225b3cddb is 1080, key is row0217/info:/1732491594828/Put/seqid=0
2024-11-24T23:39:56,863 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741874_1050 (size=14681)
2024-11-24T23:39:56,863 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741874_1050 (size=14681)
2024-11-24T23:39:56,864 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/51916c253f8145798a9843d225b3cddb
2024-11-24T23:39:56,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/51916c253f8145798a9843d225b3cddb as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/51916c253f8145798a9843d225b3cddb
2024-11-24T23:39:56,876 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/51916c253f8145798a9843d225b3cddb, entries=9, sequenceid=301, filesize=14.3 K
2024-11-24T23:39:56,877 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=12.61 KB/12912 for 1052af1b21d3f2f69648c2a80dedaae0 in 28ms, sequenceid=301, compaction requested=true
2024-11-24T23:39:56,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:56,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1052af1b21d3f2f69648c2a80dedaae0:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:39:56,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:56,878 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:56,879 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 192887 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:39:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:56,879 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1541): 1052af1b21d3f2f69648c2a80dedaae0/info is initiating minor compaction (all files)
2024-11-24T23:39:56,879 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-24T23:39:56,879 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1052af1b21d3f2f69648c2a80dedaae0/info in TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:56,880 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/9f2d8f2c4c5f4dbe9ceb45b3074d2d0d, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/f765a28969364505adc9d3bb29b688a4, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/51916c253f8145798a9843d225b3cddb] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp, totalSize=188.4 K
2024-11-24T23:39:56,880 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f2d8f2c4c5f4dbe9ceb45b3074d2d0d, keycount=143, bloomtype=ROW, size=156.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732491574317
2024-11-24T23:39:56,881 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting f765a28969364505adc9d3bb29b688a4, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732491594806
2024-11-24T23:39:56,881 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51916c253f8145798a9843d225b3cddb, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732491594828
2024-11-24T23:39:56,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/a658afa6a55742ad94e02de21bc74591 is 1080, key is row0226/info:/1732491596851/Put/seqid=0
2024-11-24T23:39:56,893 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741875_1051 (size=19013)
2024-11-24T23:39:56,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741875_1051 (size=19013)
2024-11-24T23:39:56,895 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/a658afa6a55742ad94e02de21bc74591
2024-11-24T23:39:56,898 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1052af1b21d3f2f69648c2a80dedaae0#info#compaction#90 average throughput is 42.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:56,899 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/afbc7e22d036465cafa227a940362a9f is 1080, key is row0062/info:/1732491574317/Put/seqid=0
2024-11-24T23:39:56,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/a658afa6a55742ad94e02de21bc74591 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a658afa6a55742ad94e02de21bc74591
2024-11-24T23:39:56,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741876_1052 (size=183053)
2024-11-24T23:39:56,902 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741876_1052 (size=183053)
2024-11-24T23:39:56,907 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a658afa6a55742ad94e02de21bc74591, entries=13, sequenceid=317, filesize=18.6 K
2024-11-24T23:39:56,907 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/afbc7e22d036465cafa227a940362a9f as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/afbc7e22d036465cafa227a940362a9f
2024-11-24T23:39:56,908 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=13.66 KB/13988 for 1052af1b21d3f2f69648c2a80dedaae0 in 28ms, sequenceid=317, compaction requested=false
2024-11-24T23:39:56,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46359 {}] regionserver.HRegion(8855): Flush requested on 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:56,909 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-24T23:39:56,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/6c7e292136f34b1684c3bed447185d2b is 1080, key is row0239/info:/1732491596881/Put/seqid=0
2024-11-24T23:39:56,914 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1052af1b21d3f2f69648c2a80dedaae0/info of 1052af1b21d3f2f69648c2a80dedaae0 into afbc7e22d036465cafa227a940362a9f(size=178.8 K), total size for store is 197.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:56,914 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:56,914 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0., storeName=1052af1b21d3f2f69648c2a80dedaae0/info, priority=13, startTime=1732491596878; duration=0sec
2024-11-24T23:39:56,914 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:56,914 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1052af1b21d3f2f69648c2a80dedaae0:info
2024-11-24T23:39:56,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741877_1053 (size=20092)
2024-11-24T23:39:56,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741877_1053 (size=20092)
2024-11-24T23:39:56,921 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/6c7e292136f34b1684c3bed447185d2b
2024-11-24T23:39:56,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/6c7e292136f34b1684c3bed447185d2b as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6c7e292136f34b1684c3bed447185d2b
2024-11-24T23:39:56,930 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6c7e292136f34b1684c3bed447185d2b, entries=14, sequenceid=334, filesize=19.6 K
2024-11-24T23:39:56,931 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=4.20 KB/4304 for 1052af1b21d3f2f69648c2a80dedaae0 in 22ms, sequenceid=334, compaction requested=true
2024-11-24T23:39:56,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:56,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1052af1b21d3f2f69648c2a80dedaae0:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T23:39:56,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:56,931 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T23:39:56,932 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 222158 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T23:39:56,932 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1541): 1052af1b21d3f2f69648c2a80dedaae0/info is initiating minor compaction (all files)
2024-11-24T23:39:56,932 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1052af1b21d3f2f69648c2a80dedaae0/info in TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:56,932 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/afbc7e22d036465cafa227a940362a9f, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a658afa6a55742ad94e02de21bc74591, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6c7e292136f34b1684c3bed447185d2b] into tmpdir=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp, totalSize=217.0 K
2024-11-24T23:39:56,933 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting afbc7e22d036465cafa227a940362a9f, keycount=164, bloomtype=ROW, size=178.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732491574317
2024-11-24T23:39:56,933 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting a658afa6a55742ad94e02de21bc74591, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732491596851
2024-11-24T23:39:56,933 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c7e292136f34b1684c3bed447185d2b, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732491596881
2024-11-24T23:39:56,944 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1052af1b21d3f2f69648c2a80dedaae0#info#compaction#92 average throughput is 49.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T23:39:56,944 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/a287a51e40ea4f08ba0460deb8d83c4b is 1080, key is row0062/info:/1732491574317/Put/seqid=0
2024-11-24T23:39:56,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741878_1054 (size=212365)
2024-11-24T23:39:56,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741878_1054 (size=212365)
2024-11-24T23:39:56,951 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/a287a51e40ea4f08ba0460deb8d83c4b as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a287a51e40ea4f08ba0460deb8d83c4b
2024-11-24T23:39:56,956 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1052af1b21d3f2f69648c2a80dedaae0/info of 1052af1b21d3f2f69648c2a80dedaae0 into a287a51e40ea4f08ba0460deb8d83c4b(size=207.4 K), total size for store is 207.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T23:39:56,956 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:56,956 INFO  [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0., storeName=1052af1b21d3f2f69648c2a80dedaae0/info, priority=13, startTime=1732491596931; duration=0sec
2024-11-24T23:39:56,957 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T23:39:56,957 DEBUG [RS:0;ba7f32532130:46359-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1052af1b21d3f2f69648c2a80dedaae0:info
2024-11-24T23:39:57,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:57,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:58,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:58,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:58,917 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-24T23:39:58,918 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C46359%2C1732491560986.1732491598917
2024-11-24T23:39:58,927 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:58,928 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:58,928 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:58,928 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:58,928 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:58,928 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986/ba7f32532130%2C46359%2C1732491560986.1732491561625 with entries=318, filesize=310.38 KB; new WAL /user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986/ba7f32532130%2C46359%2C1732491560986.1732491598917
2024-11-24T23:39:58,930 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34627:34627),(127.0.0.1/127.0.0.1:40115:40115)]
2024-11-24T23:39:58,930 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986/ba7f32532130%2C46359%2C1732491560986.1732491561625 is not closed yet, will try archiving it next time
2024-11-24T23:39:58,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741833_1009 (size=317837)
2024-11-24T23:39:58,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741833_1009 (size=317837)
2024-11-24T23:39:58,973 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2663dd450f9f98026c70e9ee11bb8094:

2024-11-24T23:39:58,974 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB
2024-11-24T23:39:58,981 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/info/f37caf913af244deb80341b21b04168f is 186, key is TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094./info:regioninfo/1732491577305/Put/seqid=0
2024-11-24T23:39:58,985 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741880_1056 (size=6153)
2024-11-24T23:39:58,985 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741880_1056 (size=6153)
2024-11-24T23:39:58,986 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/info/f37caf913af244deb80341b21b04168f
2024-11-24T23:39:58,991 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/.tmp/info/f37caf913af244deb80341b21b04168f as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/info/f37caf913af244deb80341b21b04168f
2024-11-24T23:39:58,998 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/info/f37caf913af244deb80341b21b04168f, entries=5, sequenceid=21, filesize=6.0 K
2024-11-24T23:39:59,000 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 25ms, sequenceid=21, compaction requested=false
2024-11-24T23:39:59,000 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-24T23:39:59,000 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1052af1b21d3f2f69648c2a80dedaae0 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-24T23:39:59,004 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/3f5b366015904ae1bf2b26517105bfb5 is 1080, key is row0253/info:/1732491596910/Put/seqid=0
2024-11-24T23:39:59,008 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741881_1057 (size=9278)
2024-11-24T23:39:59,008 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741881_1057 (size=9278)
2024-11-24T23:39:59,008 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/3f5b366015904ae1bf2b26517105bfb5
2024-11-24T23:39:59,013 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/.tmp/info/3f5b366015904ae1bf2b26517105bfb5 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3f5b366015904ae1bf2b26517105bfb5
2024-11-24T23:39:59,017 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3f5b366015904ae1bf2b26517105bfb5, entries=4, sequenceid=343, filesize=9.1 K
2024-11-24T23:39:59,018 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 1052af1b21d3f2f69648c2a80dedaae0 in 18ms, sequenceid=343, compaction requested=false
2024-11-24T23:39:59,018 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1052af1b21d3f2f69648c2a80dedaae0:

2024-11-24T23:39:59,018 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C46359%2C1732491560986.1732491599018
2024-11-24T23:39:59,022 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,022 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,023 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,023 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,023 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,023 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986/ba7f32532130%2C46359%2C1732491560986.1732491598917 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986/ba7f32532130%2C46359%2C1732491560986.1732491599018
2024-11-24T23:39:59,024 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40115:40115),(127.0.0.1/127.0.0.1:34627:34627)]
2024-11-24T23:39:59,024 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986/ba7f32532130%2C46359%2C1732491560986.1732491598917 is not closed yet, will try archiving it next time
2024-11-24T23:39:59,024 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986/ba7f32532130%2C46359%2C1732491560986.1732491561625 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/oldWALs/ba7f32532130%2C46359%2C1732491560986.1732491561625
2024-11-24T23:39:59,024 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741879_1055 (size=731)
2024-11-24T23:39:59,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741879_1055 (size=731)
2024-11-24T23:39:59,025 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-24T23:39:59,025 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/WALs/ba7f32532130,46359,1732491560986/ba7f32532130%2C46359%2C1732491560986.1732491598917 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/oldWALs/ba7f32532130%2C46359%2C1732491560986.1732491598917
2024-11-24T23:39:59,125 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T23:39:59,125 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:39:59,126 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:39:59,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:39:59,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:39:59,126 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T23:39:59,126 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T23:39:59,127 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2004771276, stopped=false
2024-11-24T23:39:59,127 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ba7f32532130,35835,1732491560786
2024-11-24T23:39:59,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:39:59,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:39:59,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:59,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:59,198 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:39:59,199 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:39:59,200 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:39:59,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:39:59,200 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:39:59,200 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:39:59,201 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba7f32532130,46359,1732491560986' *****
2024-11-24T23:39:59,201 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T23:39:59,201 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T23:39:59,201 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T23:39:59,202 INFO  [RS:0;ba7f32532130:46359 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T23:39:59,202 INFO  [RS:0;ba7f32532130:46359 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T23:39:59,202 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(3091): Received CLOSE for 2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:59,202 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(3091): Received CLOSE for 1052af1b21d3f2f69648c2a80dedaae0
2024-11-24T23:39:59,203 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(959): stopping server ba7f32532130,46359,1732491560986
2024-11-24T23:39:59,203 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2663dd450f9f98026c70e9ee11bb8094, disabling compactions & flushes
2024-11-24T23:39:59,203 INFO  [RS:0;ba7f32532130:46359 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:39:59,203 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.
2024-11-24T23:39:59,203 INFO  [RS:0;ba7f32532130:46359 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ba7f32532130:46359.
2024-11-24T23:39:59,203 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.
2024-11-24T23:39:59,203 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094. after waiting 0 ms
2024-11-24T23:39:59,203 DEBUG [RS:0;ba7f32532130:46359 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:39:59,203 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.
2024-11-24T23:39:59,203 DEBUG [RS:0;ba7f32532130:46359 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:39:59,203 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T23:39:59,203 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T23:39:59,203 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T23:39:59,203 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T23:39:59,204 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-24T23:39:59,204 DEBUG [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(1325): Online Regions={2663dd450f9f98026c70e9ee11bb8094=TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094., 1588230740=hbase:meta,,1.1588230740, 1052af1b21d3f2f69648c2a80dedaae0=TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.}
2024-11-24T23:39:59,204 DEBUG [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(1351): Waiting on 1052af1b21d3f2f69648c2a80dedaae0, 1588230740, 2663dd450f9f98026c70e9ee11bb8094
2024-11-24T23:39:59,204 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:39:59,204 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:39:59,204 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449->hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/96c490f97dd042c4bdefc1a1ff160658-bottom] to archive
2024-11-24T23:39:59,204 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:39:59,204 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:39:59,204 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:39:59,205 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T23:39:59,207 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449
2024-11-24T23:39:59,207 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ba7f32532130:35835 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-24T23:39:59,207 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-24T23:39:59,208 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-24T23:39:59,209 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:39:59,209 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:39:59,209 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491599204Running coprocessor pre-close hooks at 1732491599204Disabling compacts and flushes for region at 1732491599204Disabling writes for close at 1732491599204Writing region close event to WAL at 1732491599205 (+1 ms)Running coprocessor post-close hooks at 1732491599208 (+3 ms)Closed at 1732491599209 (+1 ms)
2024-11-24T23:39:59,209 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T23:39:59,210 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/2663dd450f9f98026c70e9ee11bb8094/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-11-24T23:39:59,211 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.
2024-11-24T23:39:59,211 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2663dd450f9f98026c70e9ee11bb8094:
Waiting for close lock at 1732491599203Running coprocessor pre-close hooks at 1732491599203Disabling compacts and flushes for region at 1732491599203Disabling writes for close at 1732491599203Writing region close event to WAL at 1732491599207 (+4 ms)Running coprocessor post-close hooks at 1732491599210 (+3 ms)Closed at 1732491599211 (+1 ms)
2024-11-24T23:39:59,211 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1732491576445.2663dd450f9f98026c70e9ee11bb8094.
2024-11-24T23:39:59,211 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1052af1b21d3f2f69648c2a80dedaae0, disabling compactions & flushes
2024-11-24T23:39:59,211 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:59,211 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:59,211 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0. after waiting 0 ms
2024-11-24T23:39:59,211 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:59,211 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449->hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/13f4128eec290306f228e54615cac449/info/96c490f97dd042c4bdefc1a1ff160658-top, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-05ad70c1cf6b44e38bdeeab50a87ceeb, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/b4937c41e81f413ca12ef11c808d7b48, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-b78366cee3a742a8b0b89625a1bb3647, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6ebff42cb0ef4791ad78ef9c6d7f9373, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/d885063f9c584259930e1f3320a5d9a4, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3812fd69754944438c95cae42aeebd8e, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a54033f2f8b74ea68799e26ed865cd82, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/f641e7f016d9405381c83883ebe7d7a7, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/10187fa2595149cdbdfc7e4668739399, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/35f5e504d761407b993be2aec36e4b87, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/1c3a3880d9394726b1c92ce255c2c1af, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/165e6431b5a74ff588076317d5d2d4cf, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/5a89f465a35042ae8cb9ad3a20d3293c, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3489b5c8c92e4f74bf2df981ee264688, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e6d908c60cf246c9ab318063fff20da3, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e694987a15304f7fa135215efa732662, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/9f2d8f2c4c5f4dbe9ceb45b3074d2d0d, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/74b316d5e07b4423893ea81ae908983e, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/f765a28969364505adc9d3bb29b688a4, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/afbc7e22d036465cafa227a940362a9f, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/51916c253f8145798a9843d225b3cddb, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a658afa6a55742ad94e02de21bc74591, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6c7e292136f34b1684c3bed447185d2b] to archive
2024-11-24T23:39:59,212 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T23:39:59,213 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/96c490f97dd042c4bdefc1a1ff160658.13f4128eec290306f228e54615cac449
2024-11-24T23:39:59,214 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-05ad70c1cf6b44e38bdeeab50a87ceeb to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-05ad70c1cf6b44e38bdeeab50a87ceeb
2024-11-24T23:39:59,215 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/b4937c41e81f413ca12ef11c808d7b48 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/b4937c41e81f413ca12ef11c808d7b48
2024-11-24T23:39:59,216 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-b78366cee3a742a8b0b89625a1bb3647 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/TestLogRolling-testLogRolling=13f4128eec290306f228e54615cac449-b78366cee3a742a8b0b89625a1bb3647
2024-11-24T23:39:59,217 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6ebff42cb0ef4791ad78ef9c6d7f9373 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6ebff42cb0ef4791ad78ef9c6d7f9373
2024-11-24T23:39:59,218 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/d885063f9c584259930e1f3320a5d9a4 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/d885063f9c584259930e1f3320a5d9a4
2024-11-24T23:39:59,219 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3812fd69754944438c95cae42aeebd8e to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3812fd69754944438c95cae42aeebd8e
2024-11-24T23:39:59,220 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a54033f2f8b74ea68799e26ed865cd82 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a54033f2f8b74ea68799e26ed865cd82
2024-11-24T23:39:59,221 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/f641e7f016d9405381c83883ebe7d7a7 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/f641e7f016d9405381c83883ebe7d7a7
2024-11-24T23:39:59,222 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/10187fa2595149cdbdfc7e4668739399 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/10187fa2595149cdbdfc7e4668739399
2024-11-24T23:39:59,223 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/35f5e504d761407b993be2aec36e4b87 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/35f5e504d761407b993be2aec36e4b87
2024-11-24T23:39:59,224 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/1c3a3880d9394726b1c92ce255c2c1af to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/1c3a3880d9394726b1c92ce255c2c1af
2024-11-24T23:39:59,224 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/165e6431b5a74ff588076317d5d2d4cf to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/165e6431b5a74ff588076317d5d2d4cf
2024-11-24T23:39:59,225 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/5a89f465a35042ae8cb9ad3a20d3293c to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/5a89f465a35042ae8cb9ad3a20d3293c
2024-11-24T23:39:59,226 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3489b5c8c92e4f74bf2df981ee264688 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/3489b5c8c92e4f74bf2df981ee264688
2024-11-24T23:39:59,227 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e6d908c60cf246c9ab318063fff20da3 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e6d908c60cf246c9ab318063fff20da3
2024-11-24T23:39:59,228 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e694987a15304f7fa135215efa732662 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/e694987a15304f7fa135215efa732662
2024-11-24T23:39:59,229 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/9f2d8f2c4c5f4dbe9ceb45b3074d2d0d to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/9f2d8f2c4c5f4dbe9ceb45b3074d2d0d
2024-11-24T23:39:59,229 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/74b316d5e07b4423893ea81ae908983e to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/74b316d5e07b4423893ea81ae908983e
2024-11-24T23:39:59,230 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/f765a28969364505adc9d3bb29b688a4 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/f765a28969364505adc9d3bb29b688a4
2024-11-24T23:39:59,231 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/afbc7e22d036465cafa227a940362a9f to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/afbc7e22d036465cafa227a940362a9f
2024-11-24T23:39:59,232 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/51916c253f8145798a9843d225b3cddb to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/51916c253f8145798a9843d225b3cddb
2024-11-24T23:39:59,233 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a658afa6a55742ad94e02de21bc74591 to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/a658afa6a55742ad94e02de21bc74591
2024-11-24T23:39:59,233 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6c7e292136f34b1684c3bed447185d2b to hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/archive/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/info/6c7e292136f34b1684c3bed447185d2b
2024-11-24T23:39:59,234 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [b4937c41e81f413ca12ef11c808d7b48=42984, 6ebff42cb0ef4791ad78ef9c6d7f9373=12516, d885063f9c584259930e1f3320a5d9a4=64714, 3812fd69754944438c95cae42aeebd8e=19000, a54033f2f8b74ea68799e26ed865cd82=19000, f641e7f016d9405381c83883ebe7d7a7=91843, 10187fa2595149cdbdfc7e4668739399=17906, 35f5e504d761407b993be2aec36e4b87=16828, 1c3a3880d9394726b1c92ce255c2c1af=113509, 165e6431b5a74ff588076317d5d2d4cf=14672, 5a89f465a35042ae8cb9ad3a20d3293c=17906, 3489b5c8c92e4f74bf2df981ee264688=138610, e6d908c60cf246c9ab318063fff20da3=16828, e694987a15304f7fa135215efa732662=14673, 9f2d8f2c4c5f4dbe9ceb45b3074d2d0d=160288, 74b316d5e07b4423893ea81ae908983e=16839, f765a28969364505adc9d3bb29b688a4=17918, afbc7e22d036465cafa227a940362a9f=183053, 51916c253f8145798a9843d225b3cddb=14681, a658afa6a55742ad94e02de21bc74591=19013, 6c7e292136f34b1684c3bed447185d2b=20092]
2024-11-24T23:39:59,237 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/data/default/TestLogRolling-testLogRolling/1052af1b21d3f2f69648c2a80dedaae0/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=130
2024-11-24T23:39:59,237 INFO  [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:59,237 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1052af1b21d3f2f69648c2a80dedaae0:
Waiting for close lock at 1732491599211Running coprocessor pre-close hooks at 1732491599211Disabling compacts and flushes for region at 1732491599211Disabling writes for close at 1732491599211Writing region close event to WAL at 1732491599234 (+23 ms)Running coprocessor post-close hooks at 1732491599237 (+3 ms)Closed at 1732491599237
2024-11-24T23:39:59,237 DEBUG [RS_CLOSE_REGION-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1732491576445.1052af1b21d3f2f69648c2a80dedaae0.
2024-11-24T23:39:59,404 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(976): stopping server ba7f32532130,46359,1732491560986; all regions closed.
2024-11-24T23:39:59,406 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,406 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,406 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,407 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,407 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,412 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741834_1010 (size=8107)
2024-11-24T23:39:59,413 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741834_1010 (size=8107)
2024-11-24T23:39:59,416 DEBUG [RS:0;ba7f32532130:46359 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/oldWALs
2024-11-24T23:39:59,416 INFO  [RS:0;ba7f32532130:46359 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C46359%2C1732491560986.meta:.meta(num 1732491561947)
2024-11-24T23:39:59,417 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,417 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,417 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,417 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,417 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,419 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741882_1058 (size=780)
2024-11-24T23:39:59,419 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741882_1058 (size=780)
2024-11-24T23:39:59,421 DEBUG [RS:0;ba7f32532130:46359 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/oldWALs
2024-11-24T23:39:59,421 INFO  [RS:0;ba7f32532130:46359 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C46359%2C1732491560986:(num 1732491599018)
2024-11-24T23:39:59,421 DEBUG [RS:0;ba7f32532130:46359 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:39:59,421 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:39:59,422 INFO  [RS:0;ba7f32532130:46359 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:39:59,422 INFO  [RS:0;ba7f32532130:46359 {}] hbase.ChoreService(370): Chore service for: regionserver/ba7f32532130:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-24T23:39:59,422 INFO  [RS:0;ba7f32532130:46359 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:39:59,422 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:39:59,422 INFO  [RS:0;ba7f32532130:46359 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46359
2024-11-24T23:39:59,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba7f32532130,46359,1732491560986
2024-11-24T23:39:59,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:39:59,428 INFO  [RS:0;ba7f32532130:46359 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:39:59,439 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba7f32532130,46359,1732491560986]
2024-11-24T23:39:59,449 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba7f32532130,46359,1732491560986 already deleted, retry=false
2024-11-24T23:39:59,449 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba7f32532130,46359,1732491560986 expired; onlineServers=0
2024-11-24T23:39:59,449 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ba7f32532130,35835,1732491560786' *****
2024-11-24T23:39:59,449 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T23:39:59,449 INFO  [M:0;ba7f32532130:35835 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:39:59,449 INFO  [M:0;ba7f32532130:35835 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:39:59,450 DEBUG [M:0;ba7f32532130:35835 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T23:39:59,450 DEBUG [M:0;ba7f32532130:35835 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T23:39:59,450 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T23:39:59,450 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491561325 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491561325,5,FailOnTimeoutGroup]
2024-11-24T23:39:59,450 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491561325 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491561325,5,FailOnTimeoutGroup]
2024-11-24T23:39:59,450 INFO  [M:0;ba7f32532130:35835 {}] hbase.ChoreService(370): Chore service for: master/ba7f32532130:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T23:39:59,450 INFO  [M:0;ba7f32532130:35835 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:39:59,450 DEBUG [M:0;ba7f32532130:35835 {}] master.HMaster(1795): Stopping service threads
2024-11-24T23:39:59,450 INFO  [M:0;ba7f32532130:35835 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T23:39:59,450 INFO  [M:0;ba7f32532130:35835 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:39:59,450 INFO  [M:0;ba7f32532130:35835 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T23:39:59,450 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T23:39:59,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T23:39:59,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:39:59,460 DEBUG [M:0;ba7f32532130:35835 {}] zookeeper.ZKUtil(347): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T23:39:59,460 WARN  [M:0;ba7f32532130:35835 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T23:39:59,461 INFO  [M:0;ba7f32532130:35835 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/.lastflushedseqids
2024-11-24T23:39:59,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741883_1059 (size=228)
2024-11-24T23:39:59,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741883_1059 (size=228)
2024-11-24T23:39:59,466 INFO  [M:0;ba7f32532130:35835 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T23:39:59,466 INFO  [M:0;ba7f32532130:35835 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T23:39:59,466 DEBUG [M:0;ba7f32532130:35835 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:39:59,466 INFO  [M:0;ba7f32532130:35835 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:59,466 DEBUG [M:0;ba7f32532130:35835 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:59,466 DEBUG [M:0;ba7f32532130:35835 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:39:59,466 DEBUG [M:0;ba7f32532130:35835 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:59,466 INFO  [M:0;ba7f32532130:35835 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.44 KB heapSize=63.39 KB
2024-11-24T23:39:59,483 INFO  [regionserver/ba7f32532130:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:39:59,485 DEBUG [M:0;ba7f32532130:35835 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/be6bcfb7fef84c7e90ef85585190895b is 82, key is hbase:meta,,1/info:regioninfo/1732491561971/Put/seqid=0
2024-11-24T23:39:59,489 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741884_1060 (size=5672)
2024-11-24T23:39:59,489 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741884_1060 (size=5672)
2024-11-24T23:39:59,490 INFO  [M:0;ba7f32532130:35835 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/be6bcfb7fef84c7e90ef85585190895b
2024-11-24T23:39:59,507 DEBUG [M:0;ba7f32532130:35835 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41e419f64f384808b03cf41ec7c0a399 is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732491562504/Put/seqid=0
2024-11-24T23:39:59,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:59,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741885_1061 (size=7091)
2024-11-24T23:39:59,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741885_1061 (size=7091)
2024-11-24T23:39:59,512 INFO  [M:0;ba7f32532130:35835 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.84 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41e419f64f384808b03cf41ec7c0a399
2024-11-24T23:39:59,516 INFO  [M:0;ba7f32532130:35835 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 41e419f64f384808b03cf41ec7c0a399
2024-11-24T23:39:59,529 DEBUG [M:0;ba7f32532130:35835 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3d253d6f50e54c64ac079c12eb0e7c91 is 69, key is ba7f32532130,46359,1732491560986/rs:state/1732491561459/Put/seqid=0
2024-11-24T23:39:59,533 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741886_1062 (size=5156)
2024-11-24T23:39:59,534 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741886_1062 (size=5156)
2024-11-24T23:39:59,534 INFO  [M:0;ba7f32532130:35835 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3d253d6f50e54c64ac079c12eb0e7c91
2024-11-24T23:39:59,539 INFO  [RS:0;ba7f32532130:46359 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:39:59,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:39:59,539 INFO  [RS:0;ba7f32532130:46359 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba7f32532130,46359,1732491560986; zookeeper connection closed.
2024-11-24T23:39:59,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46359-0x1016f3742ef0001, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:39:59,539 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@13e652dd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@13e652dd
2024-11-24T23:39:59,539 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T23:39:59,552 DEBUG [M:0;ba7f32532130:35835 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cc76b0cf22954fbf935bb46ac110ae52 is 52, key is load_balancer_on/state:d/1732491562121/Put/seqid=0
2024-11-24T23:39:59,556 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741887_1063 (size=5056)
2024-11-24T23:39:59,556 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741887_1063 (size=5056)
2024-11-24T23:39:59,558 INFO  [M:0;ba7f32532130:35835 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cc76b0cf22954fbf935bb46ac110ae52
2024-11-24T23:39:59,562 DEBUG [M:0;ba7f32532130:35835 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/be6bcfb7fef84c7e90ef85585190895b as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/be6bcfb7fef84c7e90ef85585190895b
2024-11-24T23:39:59,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:39:59,566 INFO  [M:0;ba7f32532130:35835 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/be6bcfb7fef84c7e90ef85585190895b, entries=8, sequenceid=125, filesize=5.5 K
2024-11-24T23:39:59,567 DEBUG [M:0;ba7f32532130:35835 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41e419f64f384808b03cf41ec7c0a399 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41e419f64f384808b03cf41ec7c0a399
2024-11-24T23:39:59,571 INFO  [M:0;ba7f32532130:35835 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 41e419f64f384808b03cf41ec7c0a399
2024-11-24T23:39:59,571 INFO  [M:0;ba7f32532130:35835 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41e419f64f384808b03cf41ec7c0a399, entries=13, sequenceid=125, filesize=6.9 K
2024-11-24T23:39:59,572 DEBUG [M:0;ba7f32532130:35835 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3d253d6f50e54c64ac079c12eb0e7c91 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3d253d6f50e54c64ac079c12eb0e7c91
2024-11-24T23:39:59,576 INFO  [M:0;ba7f32532130:35835 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3d253d6f50e54c64ac079c12eb0e7c91, entries=1, sequenceid=125, filesize=5.0 K
2024-11-24T23:39:59,577 DEBUG [M:0;ba7f32532130:35835 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cc76b0cf22954fbf935bb46ac110ae52 as hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/cc76b0cf22954fbf935bb46ac110ae52
2024-11-24T23:39:59,582 INFO  [M:0;ba7f32532130:35835 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44631/user/jenkins/test-data/92864d69-058f-df70-b070-e97ef4b7a25e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/cc76b0cf22954fbf935bb46ac110ae52, entries=1, sequenceid=125, filesize=4.9 K
2024-11-24T23:39:59,583 INFO  [M:0;ba7f32532130:35835 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.44 KB/52675, heapSize ~63.33 KB/64848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=125, compaction requested=false
2024-11-24T23:39:59,584 INFO  [M:0;ba7f32532130:35835 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:39:59,584 DEBUG [M:0;ba7f32532130:35835 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491599466Disabling compacts and flushes for region at 1732491599466Disabling writes for close at 1732491599466Obtaining lock to block concurrent updates at 1732491599466Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732491599466Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52675, getHeapSize=64848, getOffHeapSize=0, getCellsCount=148 at 1732491599467 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732491599467Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732491599467Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732491599485 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732491599485Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732491599494 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732491599506 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732491599506Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732491599516 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732491599529 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732491599529Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732491599538 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732491599551 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732491599551Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f7f25e1: reopening flushed file at 1732491599562 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e7ff93d: reopening flushed file at 1732491599567 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@637e8eb2: reopening flushed file at 1732491599571 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b4e3ede: reopening flushed file at 1732491599576 (+5 ms)Finished flush of dataSize ~51.44 KB/52675, heapSize ~63.33 KB/64848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=125, compaction requested=false at 1732491599583 (+7 ms)Writing region close event to WAL at 1732491599584 (+1 ms)Closed at 1732491599584
2024-11-24T23:39:59,587 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,587 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,587 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,587 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,587 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:39:59,589 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41415 is added to blk_1073741830_1006 (size=61344)
2024-11-24T23:39:59,589 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43417 is added to blk_1073741830_1006 (size=61344)
2024-11-24T23:39:59,589 INFO  [M:0;ba7f32532130:35835 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T23:39:59,589 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:39:59,589 INFO  [M:0;ba7f32532130:35835 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35835
2024-11-24T23:39:59,590 INFO  [M:0;ba7f32532130:35835 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:39:59,697 INFO  [M:0;ba7f32532130:35835 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:39:59,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:39:59,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35835-0x1016f3742ef0000, quorum=127.0.0.1:51651, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:39:59,699 WARN  [BP-1650221223-172.17.0.3-1732491558615 heartbeating to localhost/127.0.0.1:44631 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1650221223-172.17.0.3-1732491558615 (Datanode Uuid 4ea9c9fa-3b20-491b-a604-34ca24b8dbf8) service to localhost/127.0.0.1:44631
2024-11-24T23:39:59,700 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@e483a61{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:39:59,700 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee/data/data3/current/BP-1650221223-172.17.0.3-1732491558615 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:39:59,700 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee/data/data4/current/BP-1650221223-172.17.0.3-1732491558615 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:39:59,700 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ab86f9f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:39:59,700 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:39:59,700 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47c8059{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:39:59,700 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7181dda1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/hadoop.log.dir/,STOPPED}
2024-11-24T23:39:59,703 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:39:59,705 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@616d506d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:39:59,706 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bb23947{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:39:59,706 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:39:59,706 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37b300d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:39:59,706 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7cd1acca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/hadoop.log.dir/,STOPPED}
2024-11-24T23:39:59,708 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:39:59,708 WARN  [BP-1650221223-172.17.0.3-1732491558615 heartbeating to localhost/127.0.0.1:44631 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:39:59,708 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:39:59,708 WARN  [BP-1650221223-172.17.0.3-1732491558615 heartbeating to localhost/127.0.0.1:44631 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1650221223-172.17.0.3-1732491558615 (Datanode Uuid 72d03dd0-aca6-4793-b05e-cf31d8fc0d5e) service to localhost/127.0.0.1:44631
2024-11-24T23:39:59,709 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee/data/data1/current/BP-1650221223-172.17.0.3-1732491558615 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:39:59,709 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/cluster_f6db7d2a-a101-36a4-3ab0-99a6534a05ee/data/data2/current/BP-1650221223-172.17.0.3-1732491558615 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:39:59,709 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:39:59,715 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@43fb00bc{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:39:59,716 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3def21d3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:39:59,716 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:39:59,716 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2996c87f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:39:59,716 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d6e0cf8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/hadoop.log.dir/,STOPPED}
2024-11-24T23:39:59,722 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T23:39:59,751 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T23:39:59,758 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=233 (was 208)
Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:44631
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44631
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44631 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Command processor
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411)
	app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44631 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44631
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44631
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44631
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44631 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=518 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=116 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=4528 (was 3056) - AvailableMemoryMB LEAK? -
2024-11-24T23:39:59,765 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=233, OpenFileDescriptor=518, MaxFileDescriptor=1048576, SystemLoadAverage=116, ProcessCount=11, AvailableMemoryMB=4528
2024-11-24T23:39:59,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T23:39:59,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/hadoop.log.dir so I do NOT create it in target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601
2024-11-24T23:39:59,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a4546196-4221-0b78-aadc-41eef6aef455/hadoop.tmp.dir so I do NOT create it in target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601
2024-11-24T23:39:59,766 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0, deleteOnExit=true
2024-11-24T23:39:59,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T23:39:59,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/test.cache.data in system properties and HBase conf
2024-11-24T23:39:59,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T23:39:59,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/hadoop.log.dir in system properties and HBase conf
2024-11-24T23:39:59,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T23:39:59,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T23:39:59,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T23:39:59,767 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T23:39:59,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:39:59,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T23:39:59,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T23:39:59,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:39:59,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T23:39:59,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T23:39:59,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T23:39:59,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:39:59,768 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T23:39:59,768 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/nfs.dump.dir in system properties and HBase conf
2024-11-24T23:39:59,768 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/java.io.tmpdir in system properties and HBase conf
2024-11-24T23:39:59,768 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T23:39:59,768 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T23:39:59,768 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T23:39:59,781 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:40:00,188 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:40:00,191 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:40:00,192 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:40:00,192 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:40:00,192 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T23:40:00,192 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:40:00,193 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49428dc3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:40:00,193 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7043bc8e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:40:00,284 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@b404f99{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/java.io.tmpdir/jetty-localhost-37897-hadoop-hdfs-3_4_1-tests_jar-_-any-16458238308622291292/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:40:00,284 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3698cd86{HTTP/1.1, (http/1.1)}{localhost:37897}
2024-11-24T23:40:00,284 INFO  [Time-limited test {}] server.Server(415): Started @294868ms
2024-11-24T23:40:00,294 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T23:40:00,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:40:00,540 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:40:00,542 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:40:00,543 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:40:00,543 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:40:00,543 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:40:00,543 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acdff1a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:40:00,543 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40d0e54c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:40:00,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:40:00,635 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5eefcd14{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/java.io.tmpdir/jetty-localhost-37517-hadoop-hdfs-3_4_1-tests_jar-_-any-7683671069668655038/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:40:00,635 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@75728f34{HTTP/1.1, (http/1.1)}{localhost:37517}
2024-11-24T23:40:00,635 INFO  [Time-limited test {}] server.Server(415): Started @295219ms
2024-11-24T23:40:00,636 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:40:00,661 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T23:40:00,663 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T23:40:00,663 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T23:40:00,663 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T23:40:00,663 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T23:40:00,664 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3829c39e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/hadoop.log.dir/,AVAILABLE}
2024-11-24T23:40:00,664 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72757a2f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T23:40:00,773 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2269c58e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/java.io.tmpdir/jetty-localhost-45671-hadoop-hdfs-3_4_1-tests_jar-_-any-18027997117043686262/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:40:00,774 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@477d322e{HTTP/1.1, (http/1.1)}{localhost:45671}
2024-11-24T23:40:00,774 INFO  [Time-limited test {}] server.Server(415): Started @295357ms
2024-11-24T23:40:00,775 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T23:40:01,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:40:01,567 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:40:01,840 WARN  [Thread-2499 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0/data/data1/current/BP-1437415538-172.17.0.3-1732491599785/current, will proceed with Du for space computation calculation, 
2024-11-24T23:40:01,841 WARN  [Thread-2500 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0/data/data2/current/BP-1437415538-172.17.0.3-1732491599785/current, will proceed with Du for space computation calculation, 
2024-11-24T23:40:01,859 WARN  [Thread-2463 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:40:01,861 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x931b214316c27c4a with lease ID 0xe1fa5455140c3e19: Processing first storage report for DS-1702d03c-8f25-45a8-9407-f2c6ac435d30 from datanode DatanodeRegistration(127.0.0.1:42657, datanodeUuid=9a93ff89-3842-479d-aa7c-55228c7afd60, infoPort=44147, infoSecurePort=0, ipcPort=37291, storageInfo=lv=-57;cid=testClusterID;nsid=2088592359;c=1732491599785)
2024-11-24T23:40:01,861 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x931b214316c27c4a with lease ID 0xe1fa5455140c3e19: from storage DS-1702d03c-8f25-45a8-9407-f2c6ac435d30 node DatanodeRegistration(127.0.0.1:42657, datanodeUuid=9a93ff89-3842-479d-aa7c-55228c7afd60, infoPort=44147, infoSecurePort=0, ipcPort=37291, storageInfo=lv=-57;cid=testClusterID;nsid=2088592359;c=1732491599785), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:40:01,861 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x931b214316c27c4a with lease ID 0xe1fa5455140c3e19: Processing first storage report for DS-28572aa1-f049-4e05-a64c-302f9dd6065c from datanode DatanodeRegistration(127.0.0.1:42657, datanodeUuid=9a93ff89-3842-479d-aa7c-55228c7afd60, infoPort=44147, infoSecurePort=0, ipcPort=37291, storageInfo=lv=-57;cid=testClusterID;nsid=2088592359;c=1732491599785)
2024-11-24T23:40:01,861 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x931b214316c27c4a with lease ID 0xe1fa5455140c3e19: from storage DS-28572aa1-f049-4e05-a64c-302f9dd6065c node DatanodeRegistration(127.0.0.1:42657, datanodeUuid=9a93ff89-3842-479d-aa7c-55228c7afd60, infoPort=44147, infoSecurePort=0, ipcPort=37291, storageInfo=lv=-57;cid=testClusterID;nsid=2088592359;c=1732491599785), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T23:40:01,968 WARN  [Thread-2510 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0/data/data3/current/BP-1437415538-172.17.0.3-1732491599785/current, will proceed with Du for space computation calculation, 
2024-11-24T23:40:01,969 WARN  [Thread-2511 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0/data/data4/current/BP-1437415538-172.17.0.3-1732491599785/current, will proceed with Du for space computation calculation, 
2024-11-24T23:40:01,988 WARN  [Thread-2486 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T23:40:01,990 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3c6a9e52c532b107 with lease ID 0xe1fa5455140c3e1a: Processing first storage report for DS-a19e990e-4ab8-483e-b429-3eeebc98ba3d from datanode DatanodeRegistration(127.0.0.1:34575, datanodeUuid=a012458d-7b98-44e8-8210-751bf28a4f7e, infoPort=36685, infoSecurePort=0, ipcPort=42379, storageInfo=lv=-57;cid=testClusterID;nsid=2088592359;c=1732491599785)
2024-11-24T23:40:01,990 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3c6a9e52c532b107 with lease ID 0xe1fa5455140c3e1a: from storage DS-a19e990e-4ab8-483e-b429-3eeebc98ba3d node DatanodeRegistration(127.0.0.1:34575, datanodeUuid=a012458d-7b98-44e8-8210-751bf28a4f7e, infoPort=36685, infoSecurePort=0, ipcPort=42379, storageInfo=lv=-57;cid=testClusterID;nsid=2088592359;c=1732491599785), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:40:01,990 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3c6a9e52c532b107 with lease ID 0xe1fa5455140c3e1a: Processing first storage report for DS-14037810-629a-4f1f-9e39-cb4371b2302b from datanode DatanodeRegistration(127.0.0.1:34575, datanodeUuid=a012458d-7b98-44e8-8210-751bf28a4f7e, infoPort=36685, infoSecurePort=0, ipcPort=42379, storageInfo=lv=-57;cid=testClusterID;nsid=2088592359;c=1732491599785)
2024-11-24T23:40:01,990 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3c6a9e52c532b107 with lease ID 0xe1fa5455140c3e1a: from storage DS-14037810-629a-4f1f-9e39-cb4371b2302b node DatanodeRegistration(127.0.0.1:34575, datanodeUuid=a012458d-7b98-44e8-8210-751bf28a4f7e, infoPort=36685, infoSecurePort=0, ipcPort=42379, storageInfo=lv=-57;cid=testClusterID;nsid=2088592359;c=1732491599785), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T23:40:02,006 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601
2024-11-24T23:40:02,008 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0/zookeeper_0, clientPort=52699, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T23:40:02,009 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52699
2024-11-24T23:40:02,009 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:40:02,010 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:40:02,019 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:40:02,019 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741825_1001 (size=7)
2024-11-24T23:40:02,020 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a with version=8
2024-11-24T23:40:02,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43777/user/jenkins/test-data/26e6c6a6-a081-73a2-cbf1-0ccd11e035ec/hbase-staging
2024-11-24T23:40:02,022 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:40:02,022 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:40:02,022 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:40:02,022 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:40:02,022 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:40:02,022 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:40:02,022 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T23:40:02,022 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:40:02,023 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39755
2024-11-24T23:40:02,023 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39755 connecting to ZooKeeper ensemble=127.0.0.1:52699
2024-11-24T23:40:02,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:397550x0, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:40:02,082 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39755-0x1016f37e4030000 connected
2024-11-24T23:40:02,165 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:40:02,167 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:40:02,170 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:40:02,170 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a, hbase.cluster.distributed=false
2024-11-24T23:40:02,173 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:40:02,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39755
2024-11-24T23:40:02,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39755
2024-11-24T23:40:02,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39755
2024-11-24T23:40:02,175 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39755
2024-11-24T23:40:02,175 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39755
2024-11-24T23:40:02,192 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/ba7f32532130:0 server-side Connection retries=45
2024-11-24T23:40:02,192 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:40:02,192 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T23:40:02,192 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T23:40:02,192 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T23:40:02,192 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T23:40:02,192 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T23:40:02,192 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T23:40:02,193 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40175
2024-11-24T23:40:02,193 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40175 connecting to ZooKeeper ensemble=127.0.0.1:52699
2024-11-24T23:40:02,194 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:40:02,195 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:40:02,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401750x0, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T23:40:02,207 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40175-0x1016f37e4030001 connected
2024-11-24T23:40:02,207 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:40:02,208 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T23:40:02,208 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T23:40:02,208 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T23:40:02,209 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T23:40:02,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40175
2024-11-24T23:40:02,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40175
2024-11-24T23:40:02,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40175
2024-11-24T23:40:02,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40175
2024-11-24T23:40:02,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40175
2024-11-24T23:40:02,225 DEBUG [M:0;ba7f32532130:39755 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ba7f32532130:39755
2024-11-24T23:40:02,225 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ba7f32532130,39755,1732491602021
2024-11-24T23:40:02,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:40:02,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:40:02,228 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ba7f32532130,39755,1732491602021
2024-11-24T23:40:02,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:02,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T23:40:02,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:02,239 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T23:40:02,239 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ba7f32532130,39755,1732491602021 from backup master directory
2024-11-24T23:40:02,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:40:02,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ba7f32532130,39755,1732491602021
2024-11-24T23:40:02,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T23:40:02,249 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:40:02,249 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ba7f32532130,39755,1732491602021
2024-11-24T23:40:02,254 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/hbase.id] with ID: a7498434-1961-4077-a15c-ff6723db7942
2024-11-24T23:40:02,254 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/.tmp/hbase.id
2024-11-24T23:40:02,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:40:02,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741826_1002 (size=42)
2024-11-24T23:40:02,263 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/.tmp/hbase.id]:[hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/hbase.id]
2024-11-24T23:40:02,272 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:40:02,272 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T23:40:02,273 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-24T23:40:02,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:02,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:02,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:40:02,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741827_1003 (size=196)
2024-11-24T23:40:02,289 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T23:40:02,290 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T23:40:02,290 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:40:02,298 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:40:02,298 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741828_1004 (size=1189)
2024-11-24T23:40:02,299 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store
2024-11-24T23:40:02,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:40:02,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741829_1005 (size=34)
2024-11-24T23:40:02,304 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:40:02,304 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:40:02,305 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:40:02,305 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:40:02,305 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:40:02,305 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:40:02,305 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:40:02,305 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491602304Disabling compacts and flushes for region at 1732491602304Disabling writes for close at 1732491602305 (+1 ms)Writing region close event to WAL at 1732491602305Closed at 1732491602305
2024-11-24T23:40:02,305 WARN  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/.initializing
2024-11-24T23:40:02,305 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/WALs/ba7f32532130,39755,1732491602021
2024-11-24T23:40:02,307 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C39755%2C1732491602021, suffix=, logDir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/WALs/ba7f32532130,39755,1732491602021, archiveDir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/oldWALs, maxLogs=10
2024-11-24T23:40:02,307 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C39755%2C1732491602021.1732491602307
2024-11-24T23:40:02,311 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/WALs/ba7f32532130,39755,1732491602021/ba7f32532130%2C39755%2C1732491602021.1732491602307
2024-11-24T23:40:02,312 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44147:44147),(127.0.0.1/127.0.0.1:36685:36685)]
2024-11-24T23:40:02,313 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:40:02,313 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:40:02,313 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:40:02,313 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:40:02,314 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:40:02,315 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T23:40:02,315 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:02,315 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:40:02,316 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:40:02,316 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T23:40:02,316 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:02,317 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:40:02,317 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:40:02,318 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T23:40:02,318 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:02,318 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:40:02,318 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T23:40:02,319 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T23:40:02,319 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:02,319 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T23:40:02,319 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:40:02,320 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:40:02,320 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:40:02,321 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:40:02,321 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:40:02,322 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T23:40:02,323 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T23:40:02,324 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:40:02,325 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=884272, jitterRate=0.1244111955165863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T23:40:02,325 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732491602313Initializing all the Stores at 1732491602314 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491602314Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491602314Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491602314Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491602314Cleaning up temporary data from old regions at 1732491602321 (+7 ms)Region opened successfully at 1732491602325 (+4 ms)
2024-11-24T23:40:02,326 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T23:40:02,328 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a60b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:40:02,329 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T23:40:02,329 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T23:40:02,329 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T23:40:02,330 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T23:40:02,330 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T23:40:02,330 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T23:40:02,330 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T23:40:02,332 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T23:40:02,333 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T23:40:02,344 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T23:40:02,344 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T23:40:02,345 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T23:40:02,354 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T23:40:02,355 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T23:40:02,356 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T23:40:02,365 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T23:40:02,367 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T23:40:02,376 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T23:40:02,380 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T23:40:02,386 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T23:40:02,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:40:02,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T23:40:02,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:02,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:02,397 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ba7f32532130,39755,1732491602021, sessionid=0x1016f37e4030000, setting cluster-up flag (Was=false)
2024-11-24T23:40:02,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:02,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:02,449 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T23:40:02,451 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,39755,1732491602021
2024-11-24T23:40:02,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:02,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:02,502 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T23:40:02,507 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ba7f32532130,39755,1732491602021
2024-11-24T23:40:02,510 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T23:40:02,512 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T23:40:02,512 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T23:40:02,512 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T23:40:02,512 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(746): ClusterId : a7498434-1961-4077-a15c-ff6723db7942
2024-11-24T23:40:02,513 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T23:40:02,512 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ba7f32532130,39755,1732491602021
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T23:40:02,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:40:02,524 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T23:40:02,524 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:40:02,524 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T23:40:02,524 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:40:02,524 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:40:02,524 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ba7f32532130:0, corePoolSize=5, maxPoolSize=5
2024-11-24T23:40:02,524 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ba7f32532130:0, corePoolSize=10, maxPoolSize=10
2024-11-24T23:40:02,524 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,524 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:40:02,524 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,525 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732491632525
2024-11-24T23:40:02,525 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T23:40:02,526 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T23:40:02,526 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T23:40:02,526 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T23:40:02,526 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:40:02,526 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T23:40:02,526 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T23:40:02,526 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T23:40:02,526 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,526 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T23:40:02,526 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T23:40:02,526 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T23:40:02,526 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T23:40:02,526 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T23:40:02,527 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491602526,5,FailOnTimeoutGroup]
2024-11-24T23:40:02,527 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:02,527 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491602527,5,FailOnTimeoutGroup]
2024-11-24T23:40:02,527 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,527 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T23:40:02,527 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,527 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T23:40:02,527 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:40:02,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741831_1007 (size=1321)
2024-11-24T23:40:02,534 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T23:40:02,534 DEBUG [RS:0;ba7f32532130:40175 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f8c617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ba7f32532130/172.17.0.3:0
2024-11-24T23:40:02,549 DEBUG [RS:0;ba7f32532130:40175 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ba7f32532130:40175
2024-11-24T23:40:02,549 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T23:40:02,549 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T23:40:02,549 DEBUG [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T23:40:02,549 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(2659): reportForDuty to master=ba7f32532130,39755,1732491602021 with port=40175, startcode=1732491602191
2024-11-24T23:40:02,549 DEBUG [RS:0;ba7f32532130:40175 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T23:40:02,551 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41817, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T23:40:02,552 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39755 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ba7f32532130,40175,1732491602191
2024-11-24T23:40:02,552 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39755 {}] master.ServerManager(517): Registering regionserver=ba7f32532130,40175,1732491602191
2024-11-24T23:40:02,553 DEBUG [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a
2024-11-24T23:40:02,553 DEBUG [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33485
2024-11-24T23:40:02,553 DEBUG [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T23:40:02,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:40:02,565 DEBUG [RS:0;ba7f32532130:40175 {}] zookeeper.ZKUtil(111): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ba7f32532130,40175,1732491602191
2024-11-24T23:40:02,565 WARN  [RS:0;ba7f32532130:40175 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T23:40:02,566 INFO  [RS:0;ba7f32532130:40175 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:40:02,566 DEBUG [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/WALs/ba7f32532130,40175,1732491602191
2024-11-24T23:40:02,566 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ba7f32532130,40175,1732491602191]
2024-11-24T23:40:02,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:40:02,570 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T23:40:02,571 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T23:40:02,572 INFO  [RS:0;ba7f32532130:40175 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T23:40:02,572 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,572 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T23:40:02,573 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T23:40:02,573 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,573 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,573 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,573 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,573 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,573 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,573 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ba7f32532130:0, corePoolSize=2, maxPoolSize=2
2024-11-24T23:40:02,574 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,574 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,574 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,574 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,574 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,574 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ba7f32532130:0, corePoolSize=1, maxPoolSize=1
2024-11-24T23:40:02,574 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:40:02,574 DEBUG [RS:0;ba7f32532130:40175 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ba7f32532130:0, corePoolSize=3, maxPoolSize=3
2024-11-24T23:40:02,574 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,574 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,574 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,574 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,575 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,575 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,40175,1732491602191-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:40:02,591 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T23:40:02,591 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,40175,1732491602191-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,591 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,591 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.Replication(171): ba7f32532130,40175,1732491602191 started
2024-11-24T23:40:02,603 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:02,603 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(1482): Serving as ba7f32532130,40175,1732491602191, RpcServer on ba7f32532130/172.17.0.3:40175, sessionid=0x1016f37e4030001
2024-11-24T23:40:02,603 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T23:40:02,603 DEBUG [RS:0;ba7f32532130:40175 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ba7f32532130,40175,1732491602191
2024-11-24T23:40:02,603 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,40175,1732491602191'
2024-11-24T23:40:02,603 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T23:40:02,604 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T23:40:02,604 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T23:40:02,604 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T23:40:02,604 DEBUG [RS:0;ba7f32532130:40175 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ba7f32532130,40175,1732491602191
2024-11-24T23:40:02,604 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ba7f32532130,40175,1732491602191'
2024-11-24T23:40:02,604 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T23:40:02,604 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T23:40:02,605 DEBUG [RS:0;ba7f32532130:40175 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T23:40:02,605 INFO  [RS:0;ba7f32532130:40175 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T23:40:02,605 INFO  [RS:0;ba7f32532130:40175 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T23:40:02,708 INFO  [RS:0;ba7f32532130:40175 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C40175%2C1732491602191, suffix=, logDir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/WALs/ba7f32532130,40175,1732491602191, archiveDir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/oldWALs, maxLogs=32
2024-11-24T23:40:02,709 INFO  [RS:0;ba7f32532130:40175 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C40175%2C1732491602191.1732491602709
2024-11-24T23:40:02,718 INFO  [RS:0;ba7f32532130:40175 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/WALs/ba7f32532130,40175,1732491602191/ba7f32532130%2C40175%2C1732491602191.1732491602709
2024-11-24T23:40:02,719 DEBUG [RS:0;ba7f32532130:40175 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44147:44147),(127.0.0.1/127.0.0.1:36685:36685)]
2024-11-24T23:40:02,935 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T23:40:02,935 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a
2024-11-24T23:40:02,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741833_1009 (size=32)
2024-11-24T23:40:02,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741833_1009 (size=32)
2024-11-24T23:40:02,945 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:40:02,947 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:40:02,948 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:40:02,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:02,948 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:40:02,948 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:40:02,949 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:40:02,949 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:02,950 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:40:02,950 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:40:02,951 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:40:02,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:02,951 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:40:02,951 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:40:02,953 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:40:02,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:02,953 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:40:02,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:40:02,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/1588230740
2024-11-24T23:40:02,955 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/1588230740
2024-11-24T23:40:02,956 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:40:02,956 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:40:02,957 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:40:02,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:40:02,961 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T23:40:02,961 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=707358, jitterRate=-0.1005479097366333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:40:02,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732491602945Initializing all the Stores at 1732491602946 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491602946Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491602946Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491602946Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491602946Cleaning up temporary data from old regions at 1732491602957 (+11 ms)Region opened successfully at 1732491602962 (+5 ms)
2024-11-24T23:40:02,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:40:02,962 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:40:02,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:40:02,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:40:02,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:40:02,962 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:40:02,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491602962Disabling compacts and flushes for region at 1732491602962Disabling writes for close at 1732491602962Writing region close event to WAL at 1732491602962Closed at 1732491602962
2024-11-24T23:40:02,963 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:40:02,963 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T23:40:02,963 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T23:40:02,964 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:40:02,965 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T23:40:03,115 DEBUG [ba7f32532130:39755 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T23:40:03,117 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ba7f32532130,40175,1732491602191
2024-11-24T23:40:03,120 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,40175,1732491602191, state=OPENING
2024-11-24T23:40:03,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:40:03,144 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T23:40:03,145 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-24T23:40:03,165 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T23:40:03,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:03,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:03,178 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T23:40:03,178 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,40175,1732491602191}]
2024-11-24T23:40:03,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:40:03,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:40:03,334 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T23:40:03,339 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52381, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T23:40:03,345 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T23:40:03,345 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:40:03,347 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ba7f32532130%2C40175%2C1732491602191.meta, suffix=.meta, logDir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/WALs/ba7f32532130,40175,1732491602191, archiveDir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/oldWALs, maxLogs=32
2024-11-24T23:40:03,347 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor ba7f32532130%2C40175%2C1732491602191.meta.1732491603347.meta
2024-11-24T23:40:03,352 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/WALs/ba7f32532130,40175,1732491602191/ba7f32532130%2C40175%2C1732491602191.meta.1732491603347.meta
2024-11-24T23:40:03,354 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44147:44147),(127.0.0.1/127.0.0.1:36685:36685)]
2024-11-24T23:40:03,356 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T23:40:03,356 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T23:40:03,356 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T23:40:03,357 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T23:40:03,357 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T23:40:03,357 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T23:40:03,357 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T23:40:03,357 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T23:40:03,358 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T23:40:03,359 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T23:40:03,359 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:03,359 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:40:03,359 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T23:40:03,360 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T23:40:03,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:03,361 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:40:03,361 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T23:40:03,361 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T23:40:03,362 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:03,362 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:40:03,362 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T23:40:03,363 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T23:40:03,363 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T23:40:03,363 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T23:40:03,363 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T23:40:03,364 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/1588230740
2024-11-24T23:40:03,365 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/1588230740
2024-11-24T23:40:03,366 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T23:40:03,366 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T23:40:03,366 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T23:40:03,368 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T23:40:03,368 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=728407, jitterRate=-0.07378333806991577}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T23:40:03,368 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T23:40:03,369 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732491603357Writing region info on filesystem at 1732491603357Initializing all the Stores at 1732491603358 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491603358Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491603358Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732491603358Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732491603358Cleaning up temporary data from old regions at 1732491603366 (+8 ms)Running coprocessor post-open hooks at 1732491603368 (+2 ms)Region opened successfully at 1732491603369 (+1 ms)
2024-11-24T23:40:03,370 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732491603333
2024-11-24T23:40:03,372 DEBUG [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T23:40:03,372 INFO  [RS_OPEN_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T23:40:03,372 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ba7f32532130,40175,1732491602191
2024-11-24T23:40:03,373 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ba7f32532130,40175,1732491602191, state=OPEN
2024-11-24T23:40:03,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:40:03,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T23:40:03,419 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ba7f32532130,40175,1732491602191
2024-11-24T23:40:03,419 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:40:03,419 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T23:40:03,422 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T23:40:03,422 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ba7f32532130,40175,1732491602191 in 241 msec
2024-11-24T23:40:03,426 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T23:40:03,426 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 460 msec
2024-11-24T23:40:03,427 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T23:40:03,427 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T23:40:03,429 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:40:03,429 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,40175,1732491602191, seqNum=-1]
2024-11-24T23:40:03,429 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:40:03,431 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52837, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:40:03,436 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 924 msec
2024-11-24T23:40:03,436 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732491603436, completionTime=-1
2024-11-24T23:40:03,436 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T23:40:03,436 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T23:40:03,438 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T23:40:03,438 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732491663438
2024-11-24T23:40:03,438 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732491723438
2024-11-24T23:40:03,438 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-24T23:40:03,438 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39755,1732491602021-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:03,438 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39755,1732491602021-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:03,438 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39755,1732491602021-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:03,438 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ba7f32532130:39755, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:03,438 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:03,439 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:03,440 DEBUG [master/ba7f32532130:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T23:40:03,442 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.193sec
2024-11-24T23:40:03,442 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T23:40:03,442 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T23:40:03,442 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T23:40:03,442 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T23:40:03,442 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T23:40:03,442 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39755,1732491602021-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T23:40:03,442 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39755,1732491602021-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T23:40:03,445 DEBUG [master/ba7f32532130:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T23:40:03,445 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T23:40:03,445 INFO  [master/ba7f32532130:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ba7f32532130,39755,1732491602021-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T23:40:03,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c6a0fbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:40:03,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:40:03,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ba7f32532130,39755,-1 for getting cluster id
2024-11-24T23:40:03,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T23:40:03,515 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7498434-1961-4077-a15c-ff6723db7942'
2024-11-24T23:40:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T23:40:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7498434-1961-4077-a15c-ff6723db7942"

2024-11-24T23:40:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cfe972c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:40:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ba7f32532130,39755,-1]
2024-11-24T23:40:03,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T23:40:03,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:40:03,517 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T23:40:03,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d971f20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T23:40:03,518 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T23:40:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ba7f32532130,40175,1732491602191, seqNum=-1]
2024-11-24T23:40:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T23:40:03,520 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T23:40:03,522 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ba7f32532130,39755,1732491602021
2024-11-24T23:40:03,522 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T23:40:03,525 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T23:40:03,525 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T23:40:03,528 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/WALs/test.com,8080,1, archiveDir=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/oldWALs, maxLogs=32
2024-11-24T23:40:03,529 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732491603528
2024-11-24T23:40:03,534 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/WALs/test.com,8080,1/test.com%2C8080%2C1.1732491603528
2024-11-24T23:40:03,535 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44147:44147),(127.0.0.1/127.0.0.1:36685:36685)]
2024-11-24T23:40:03,536 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732491603536
2024-11-24T23:40:03,542 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:03,542 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:03,542 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:03,542 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:03,542 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:03,542 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/WALs/test.com,8080,1/test.com%2C8080%2C1.1732491603528 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/WALs/test.com,8080,1/test.com%2C8080%2C1.1732491603536
2024-11-24T23:40:03,543 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36685:36685),(127.0.0.1/127.0.0.1:44147:44147)]
2024-11-24T23:40:03,543 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/WALs/test.com,8080,1/test.com%2C8080%2C1.1732491603528 is not closed yet, will try archiving it next time
2024-11-24T23:40:03,543 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:03,543 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:03,543 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:03,543 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:03,543 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:03,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741835_1011 (size=93)
2024-11-24T23:40:03,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741835_1011 (size=93)
2024-11-24T23:40:03,544 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/WALs/test.com,8080,1/test.com%2C8080%2C1.1732491603528 to hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/oldWALs/test.com%2C8080%2C1.1732491603528
2024-11-24T23:40:03,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741836_1012 (size=93)
2024-11-24T23:40:03,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741836_1012 (size=93)
2024-11-24T23:40:03,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:40:03,950 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/oldWALs
2024-11-24T23:40:03,950 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1732491603536)
2024-11-24T23:40:03,950 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T23:40:03,951 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:40:03,951 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:40:03,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:40:03,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:40:03,951 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T23:40:03,952 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T23:40:03,952 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=835358000, stopped=false
2024-11-24T23:40:03,952 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ba7f32532130,39755,1732491602021
2024-11-24T23:40:03,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:40:03,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T23:40:03,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:03,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:03,997 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:40:03,997 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T23:40:03,997 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:40:03,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:40:03,997 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:40:03,997 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T23:40:03,997 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ba7f32532130,40175,1732491602191' *****
2024-11-24T23:40:03,997 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T23:40:03,997 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T23:40:03,998 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T23:40:03,998 INFO  [RS:0;ba7f32532130:40175 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T23:40:03,998 INFO  [RS:0;ba7f32532130:40175 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T23:40:03,998 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(959): stopping server ba7f32532130,40175,1732491602191
2024-11-24T23:40:03,998 INFO  [RS:0;ba7f32532130:40175 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:40:03,998 INFO  [RS:0;ba7f32532130:40175 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ba7f32532130:40175.
2024-11-24T23:40:03,998 DEBUG [RS:0;ba7f32532130:40175 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T23:40:03,998 DEBUG [RS:0;ba7f32532130:40175 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:40:03,998 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T23:40:03,998 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T23:40:03,998 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T23:40:03,998 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T23:40:03,998 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-24T23:40:03,998 DEBUG [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-24T23:40:03,998 DEBUG [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-24T23:40:03,998 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T23:40:03,998 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T23:40:03,998 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T23:40:03,998 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T23:40:03,998 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T23:40:03,999 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-24T23:40:04,013 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/1588230740/.tmp/ns/d80dece12baf44548974f058159d8fd9 is 43, key is default/ns:d/1732491603431/Put/seqid=0
2024-11-24T23:40:04,016 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741837_1013 (size=5153)
2024-11-24T23:40:04,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741837_1013 (size=5153)
2024-11-24T23:40:04,017 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/1588230740/.tmp/ns/d80dece12baf44548974f058159d8fd9
2024-11-24T23:40:04,022 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/1588230740/.tmp/ns/d80dece12baf44548974f058159d8fd9 as hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/1588230740/ns/d80dece12baf44548974f058159d8fd9
2024-11-24T23:40:04,027 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/1588230740/ns/d80dece12baf44548974f058159d8fd9, entries=2, sequenceid=6, filesize=5.0 K
2024-11-24T23:40:04,028 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false
2024-11-24T23:40:04,028 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-24T23:40:04,033 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-24T23:40:04,033 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T23:40:04,033 INFO  [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T23:40:04,033 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732491603998Running coprocessor pre-close hooks at 1732491603998Disabling compacts and flushes for region at 1732491603998Disabling writes for close at 1732491603998Obtaining lock to block concurrent updates at 1732491603999 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732491603999Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732491603999Flushing stores of hbase:meta,,1.1588230740 at 1732491603999Flushing 1588230740/ns: creating writer at 1732491603999Flushing 1588230740/ns: appending metadata at 1732491604012 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732491604012Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b355ba2: reopening flushed file at 1732491604022 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false at 1732491604028 (+6 ms)Writing region close event to WAL at 1732491604029 (+1 ms)Running coprocessor post-close hooks at 1732491604033 (+4 ms)Closed at 1732491604033
2024-11-24T23:40:04,034 DEBUG [RS_CLOSE_META-regionserver/ba7f32532130:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T23:40:04,199 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(976): stopping server ba7f32532130,40175,1732491602191; all regions closed.
2024-11-24T23:40:04,200 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,200 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,200 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,201 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,201 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741834_1010 (size=1152)
2024-11-24T23:40:04,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741834_1010 (size=1152)
2024-11-24T23:40:04,209 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,210 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,210 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,210 DEBUG [RS:0;ba7f32532130:40175 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/oldWALs
2024-11-24T23:40:04,210 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,210 INFO  [RS:0;ba7f32532130:40175 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C40175%2C1732491602191.meta:.meta(num 1732491603347)
2024-11-24T23:40:04,210 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,211 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,211 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,211 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,211 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,211 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,211 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,211 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,212 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,212 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,212 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,215 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741832_1008 (size=93)
2024-11-24T23:40:04,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741832_1008 (size=93)
2024-11-24T23:40:04,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,231 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,233 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,234 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,234 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,235 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,46117,1732491419885/ba7f32532130%2C46117%2C1732491419885.1732491420238
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:40:04,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46275/user/jenkins/test-data/0ebaf534-571e-d1d2-e303-43702a469bc7/WALs/ba7f32532130,34929,1732491418629/ba7f32532130%2C34929%2C1732491418629.meta.1732491419738.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T23:40:04,575 INFO  [regionserver/ba7f32532130:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:40:04,622 DEBUG [RS:0;ba7f32532130:40175 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/oldWALs
2024-11-24T23:40:04,622 INFO  [RS:0;ba7f32532130:40175 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog ba7f32532130%2C40175%2C1732491602191:(num 1732491602709)
2024-11-24T23:40:04,622 DEBUG [RS:0;ba7f32532130:40175 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T23:40:04,622 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T23:40:04,622 INFO  [RS:0;ba7f32532130:40175 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:40:04,623 INFO  [RS:0;ba7f32532130:40175 {}] hbase.ChoreService(370): Chore service for: regionserver/ba7f32532130:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T23:40:04,623 INFO  [RS:0;ba7f32532130:40175 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:40:04,623 INFO  [regionserver/ba7f32532130:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:40:04,623 INFO  [RS:0;ba7f32532130:40175 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40175
2024-11-24T23:40:04,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ba7f32532130,40175,1732491602191
2024-11-24T23:40:04,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T23:40:04,671 INFO  [RS:0;ba7f32532130:40175 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:40:04,681 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ba7f32532130,40175,1732491602191]
2024-11-24T23:40:04,691 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ba7f32532130,40175,1732491602191 already deleted, retry=false
2024-11-24T23:40:04,691 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ba7f32532130,40175,1732491602191 expired; onlineServers=0
2024-11-24T23:40:04,691 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ba7f32532130,39755,1732491602021' *****
2024-11-24T23:40:04,691 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T23:40:04,691 INFO  [M:0;ba7f32532130:39755 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T23:40:04,691 INFO  [M:0;ba7f32532130:39755 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T23:40:04,691 DEBUG [M:0;ba7f32532130:39755 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T23:40:04,691 DEBUG [M:0;ba7f32532130:39755 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T23:40:04,691 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T23:40:04,691 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491602526 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.large.0-1732491602526,5,FailOnTimeoutGroup]
2024-11-24T23:40:04,691 DEBUG [master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491602527 {}] cleaner.HFileCleaner(306): Exit Thread[master/ba7f32532130:0:becomeActiveMaster-HFileCleaner.small.0-1732491602527,5,FailOnTimeoutGroup]
2024-11-24T23:40:04,692 INFO  [M:0;ba7f32532130:39755 {}] hbase.ChoreService(370): Chore service for: master/ba7f32532130:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T23:40:04,692 INFO  [M:0;ba7f32532130:39755 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T23:40:04,692 DEBUG [M:0;ba7f32532130:39755 {}] master.HMaster(1795): Stopping service threads
2024-11-24T23:40:04,692 INFO  [M:0;ba7f32532130:39755 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T23:40:04,692 INFO  [M:0;ba7f32532130:39755 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T23:40:04,692 INFO  [M:0;ba7f32532130:39755 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T23:40:04,692 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T23:40:04,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T23:40:04,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T23:40:04,702 DEBUG [M:0;ba7f32532130:39755 {}] zookeeper.ZKUtil(347): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T23:40:04,702 WARN  [M:0;ba7f32532130:39755 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T23:40:04,702 INFO  [M:0;ba7f32532130:39755 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/.lastflushedseqids
2024-11-24T23:40:04,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741838_1014 (size=99)
2024-11-24T23:40:04,707 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741838_1014 (size=99)
2024-11-24T23:40:04,707 INFO  [M:0;ba7f32532130:39755 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T23:40:04,707 INFO  [M:0;ba7f32532130:39755 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T23:40:04,708 DEBUG [M:0;ba7f32532130:39755 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T23:40:04,708 INFO  [M:0;ba7f32532130:39755 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:40:04,708 DEBUG [M:0;ba7f32532130:39755 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:40:04,708 DEBUG [M:0;ba7f32532130:39755 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T23:40:04,708 DEBUG [M:0;ba7f32532130:39755 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:40:04,708 INFO  [M:0;ba7f32532130:39755 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-24T23:40:04,724 DEBUG [M:0;ba7f32532130:39755 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6c099a74ea1047c78874b48c231892ac is 82, key is hbase:meta,,1/info:regioninfo/1732491603372/Put/seqid=0
2024-11-24T23:40:04,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741839_1015 (size=5672)
2024-11-24T23:40:04,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741839_1015 (size=5672)
2024-11-24T23:40:04,728 INFO  [M:0;ba7f32532130:39755 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6c099a74ea1047c78874b48c231892ac
2024-11-24T23:40:04,739 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T23:40:04,740 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,740 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,740 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,741 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,741 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,741 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,742 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,742 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,742 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,742 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,753 DEBUG [M:0;ba7f32532130:39755 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0516da679ab427092fb4941762e39c3 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732491603435/Put/seqid=0
2024-11-24T23:40:04,757 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741840_1016 (size=5275)
2024-11-24T23:40:04,757 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741840_1016 (size=5275)
2024-11-24T23:40:04,758 INFO  [M:0;ba7f32532130:39755 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0516da679ab427092fb4941762e39c3
2024-11-24T23:40:04,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,766 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,769 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,769 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,769 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T23:40:04,776 DEBUG [M:0;ba7f32532130:39755 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b2b3e65f53649d8b599ed5c30d60c14 is 69, key is ba7f32532130,40175,1732491602191/rs:state/1732491602552/Put/seqid=0
2024-11-24T23:40:04,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741841_1017 (size=5156)
2024-11-24T23:40:04,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741841_1017 (size=5156)
2024-11-24T23:40:04,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:40:04,781 INFO  [M:0;ba7f32532130:39755 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b2b3e65f53649d8b599ed5c30d60c14
2024-11-24T23:40:04,781 INFO  [RS:0;ba7f32532130:40175 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:40:04,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40175-0x1016f37e4030001, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:40:04,781 INFO  [RS:0;ba7f32532130:40175 {}] regionserver.HRegionServer(1031): Exiting; stopping=ba7f32532130,40175,1732491602191; zookeeper connection closed.
2024-11-24T23:40:04,782 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@52b14602 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@52b14602
2024-11-24T23:40:04,782 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T23:40:04,797 DEBUG [M:0;ba7f32532130:39755 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/52b0776ee085438299c6c65a5e26931c is 52, key is load_balancer_on/state:d/1732491603524/Put/seqid=0
2024-11-24T23:40:04,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741842_1018 (size=5056)
2024-11-24T23:40:04,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741842_1018 (size=5056)
2024-11-24T23:40:04,802 INFO  [M:0;ba7f32532130:39755 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/52b0776ee085438299c6c65a5e26931c
2024-11-24T23:40:04,806 DEBUG [M:0;ba7f32532130:39755 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6c099a74ea1047c78874b48c231892ac as hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6c099a74ea1047c78874b48c231892ac
2024-11-24T23:40:04,810 INFO  [M:0;ba7f32532130:39755 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6c099a74ea1047c78874b48c231892ac, entries=8, sequenceid=29, filesize=5.5 K
2024-11-24T23:40:04,811 DEBUG [M:0;ba7f32532130:39755 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0516da679ab427092fb4941762e39c3 as hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e0516da679ab427092fb4941762e39c3
2024-11-24T23:40:04,815 INFO  [M:0;ba7f32532130:39755 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e0516da679ab427092fb4941762e39c3, entries=3, sequenceid=29, filesize=5.2 K
2024-11-24T23:40:04,816 DEBUG [M:0;ba7f32532130:39755 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b2b3e65f53649d8b599ed5c30d60c14 as hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3b2b3e65f53649d8b599ed5c30d60c14
2024-11-24T23:40:04,820 INFO  [M:0;ba7f32532130:39755 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3b2b3e65f53649d8b599ed5c30d60c14, entries=1, sequenceid=29, filesize=5.0 K
2024-11-24T23:40:04,821 DEBUG [M:0;ba7f32532130:39755 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/52b0776ee085438299c6c65a5e26931c as hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/52b0776ee085438299c6c65a5e26931c
2024-11-24T23:40:04,825 INFO  [M:0;ba7f32532130:39755 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33485/user/jenkins/test-data/49c6936a-9af5-f54b-2cae-2e7dba96c01a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/52b0776ee085438299c6c65a5e26931c, entries=1, sequenceid=29, filesize=4.9 K
2024-11-24T23:40:04,825 INFO  [M:0;ba7f32532130:39755 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=29, compaction requested=false
2024-11-24T23:40:04,827 INFO  [M:0;ba7f32532130:39755 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T23:40:04,827 DEBUG [M:0;ba7f32532130:39755 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732491604707Disabling compacts and flushes for region at 1732491604707Disabling writes for close at 1732491604708 (+1 ms)Obtaining lock to block concurrent updates at 1732491604708Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732491604708Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732491604708Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732491604709 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732491604709Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732491604724 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732491604724Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732491604732 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732491604752 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732491604752Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732491604763 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732491604776 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732491604776Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732491604785 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732491604797 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732491604797Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@642e1bd6: reopening flushed file at 1732491604805 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8d4cedd: reopening flushed file at 1732491604810 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3df3e1cd: reopening flushed file at 1732491604815 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c3fc20f: reopening flushed file at 1732491604820 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=29, compaction requested=false at 1732491604825 (+5 ms)Writing region close event to WAL at 1732491604827 (+2 ms)Closed at 1732491604827
2024-11-24T23:40:04,827 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,827 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,827 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,827 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,827 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T23:40:04,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34575 is added to blk_1073741830_1006 (size=10311)
2024-11-24T23:40:04,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42657 is added to blk_1073741830_1006 (size=10311)
2024-11-24T23:40:04,830 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T23:40:04,830 INFO  [M:0;ba7f32532130:39755 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T23:40:04,830 INFO  [M:0;ba7f32532130:39755 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39755
2024-11-24T23:40:04,830 INFO  [M:0;ba7f32532130:39755 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T23:40:04,939 INFO  [M:0;ba7f32532130:39755 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T23:40:04,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:40:04,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39755-0x1016f37e4030000, quorum=127.0.0.1:52699, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T23:40:04,944 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2269c58e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:40:04,944 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@477d322e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:40:04,944 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:40:04,944 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72757a2f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:40:04,944 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3829c39e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/hadoop.log.dir/,STOPPED}
2024-11-24T23:40:04,946 WARN  [BP-1437415538-172.17.0.3-1732491599785 heartbeating to localhost/127.0.0.1:33485 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:40:04,946 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:40:04,946 WARN  [BP-1437415538-172.17.0.3-1732491599785 heartbeating to localhost/127.0.0.1:33485 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1437415538-172.17.0.3-1732491599785 (Datanode Uuid a012458d-7b98-44e8-8210-751bf28a4f7e) service to localhost/127.0.0.1:33485
2024-11-24T23:40:04,946 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:40:04,946 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0/data/data3/current/BP-1437415538-172.17.0.3-1732491599785 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:40:04,947 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0/data/data4/current/BP-1437415538-172.17.0.3-1732491599785 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:40:04,947 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:40:04,953 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5eefcd14{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T23:40:04,954 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@75728f34{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:40:04,954 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:40:04,954 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40d0e54c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:40:04,954 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acdff1a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/hadoop.log.dir/,STOPPED}
2024-11-24T23:40:04,955 WARN  [BP-1437415538-172.17.0.3-1732491599785 heartbeating to localhost/127.0.0.1:33485 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T23:40:04,955 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T23:40:04,955 WARN  [BP-1437415538-172.17.0.3-1732491599785 heartbeating to localhost/127.0.0.1:33485 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1437415538-172.17.0.3-1732491599785 (Datanode Uuid 9a93ff89-3842-479d-aa7c-55228c7afd60) service to localhost/127.0.0.1:33485
2024-11-24T23:40:04,955 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T23:40:04,955 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0/data/data1/current/BP-1437415538-172.17.0.3-1732491599785 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:40:04,956 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/cluster_bfdbfa64-f40d-06b9-7be5-9253db3c96d0/data/data2/current/BP-1437415538-172.17.0.3-1732491599785 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T23:40:04,956 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T23:40:04,960 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@b404f99{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T23:40:04,960 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3698cd86{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T23:40:04,960 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T23:40:04,960 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7043bc8e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T23:40:04,960 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49428dc3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d3a4f0c-1c07-37ac-5d9b-eecf4254c601/hadoop.log.dir/,STOPPED}
2024-11-24T23:40:04,965 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T23:40:04,982 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T23:40:04,989 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=271 (was 233)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:33485
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33485 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:33485
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33485
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33485 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33485
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33485 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33485
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=534 (was 518) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 116) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4518 (was 4528)