2024-12-16 00:11:21,960 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc
2024-12-16 00:11:21,972 main DEBUG Took 0.009955 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-16 00:11:21,972 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-16 00:11:21,973 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-16 00:11:21,974 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-16 00:11:21,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:21,981 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-16 00:11:21,992 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:21,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:21,994 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:21,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:21,995 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:21,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:21,997 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:21,997 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:21,998 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:21,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:21,999 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:21,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:22,000 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:22,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:22,001 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:22,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:22,002 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:22,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:22,003 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:22,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:22,004 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:22,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:22,005 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:22,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-16 00:11:22,006 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:22,006 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-16 00:11:22,008 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-16 00:11:22,010 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-16 00:11:22,012 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-16 00:11:22,013 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-16 00:11:22,015 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-16 00:11:22,015 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-16 00:11:22,024 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-16 00:11:22,027 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-16 00:11:22,029 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-16 00:11:22,029 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-16 00:11:22,030 main DEBUG createAppenders(={Console})
2024-12-16 00:11:22,031 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc initialized
2024-12-16 00:11:22,031 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc
2024-12-16 00:11:22,032 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc OK.
2024-12-16 00:11:22,032 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-16 00:11:22,033 main DEBUG OutputStream closed
2024-12-16 00:11:22,033 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-16 00:11:22,034 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-16 00:11:22,034 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@53ce1329 OK
2024-12-16 00:11:22,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-16 00:11:22,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-16 00:11:22,118 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-16 00:11:22,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-16 00:11:22,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-16 00:11:22,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-16 00:11:22,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-16 00:11:22,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-16 00:11:22,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-16 00:11:22,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-16 00:11:22,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-16 00:11:22,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-16 00:11:22,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-16 00:11:22,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-16 00:11:22,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-16 00:11:22,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-16 00:11:22,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-16 00:11:22,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-16 00:11:22,126 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-16 00:11:22,126 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-12-16 00:11:22,126 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-16 00:11:22,127 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-12-16T00:11:22,333 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef
2024-12-16 00:11:22,336 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-16 00:11:22,336 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-16T00:11:22,343 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-12-16T00:11:22,374 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=286, MaxFileDescriptor=1048576, SystemLoadAverage=142, ProcessCount=11, AvailableMemoryMB=13068
2024-12-16T00:11:22,377 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-16T00:11:22,379 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54, deleteOnExit=true
2024-12-16T00:11:22,379 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-16T00:11:22,380 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/test.cache.data in system properties and HBase conf
2024-12-16T00:11:22,380 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/hadoop.tmp.dir in system properties and HBase conf
2024-12-16T00:11:22,381 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/hadoop.log.dir in system properties and HBase conf
2024-12-16T00:11:22,381 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-16T00:11:22,382 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-16T00:11:22,382 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-16T00:11:22,452 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-16T00:11:22,522 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-16T00:11:22,526 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:11:22,526 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:11:22,526 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-16T00:11:22,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:11:22,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-16T00:11:22,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-16T00:11:22,528 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:11:22,528 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:11:22,528 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-16T00:11:22,529 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/nfs.dump.dir in system properties and HBase conf
2024-12-16T00:11:22,529 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/java.io.tmpdir in system properties and HBase conf
2024-12-16T00:11:22,529 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:11:22,529 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-16T00:11:22,530 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-16T00:11:22,940 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:11:23,508 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-16T00:11:23,566 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2230ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-16T00:11:23,621 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:11:23,676 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:11:23,696 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:11:23,696 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:11:23,697 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:11:23,710 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:11:23,712 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17bd5227{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:11:23,713 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fabcd45{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:11:23,875 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@805400f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/java.io.tmpdir/jetty-localhost-41511-hadoop-hdfs-3_4_1-tests_jar-_-any-13608012920845167023/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:11:23,882 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6567b82c{HTTP/1.1, (http/1.1)}{localhost:41511}
2024-12-16T00:11:23,882 INFO  [Time-limited test {}] server.Server(415): Started @2547ms
2024-12-16T00:11:23,906 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:11:24,396 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:11:24,402 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:11:24,403 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:11:24,403 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:11:24,404 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-16T00:11:24,404 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e40bccb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:11:24,405 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@44404b82{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:11:24,496 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64b46cf2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/java.io.tmpdir/jetty-localhost-33845-hadoop-hdfs-3_4_1-tests_jar-_-any-353614301532014941/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:11:24,497 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1872bb43{HTTP/1.1, (http/1.1)}{localhost:33845}
2024-12-16T00:11:24,497 INFO  [Time-limited test {}] server.Server(415): Started @3162ms
2024-12-16T00:11:24,543 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:11:24,638 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:11:24,642 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:11:24,652 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:11:24,652 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:11:24,652 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-16T00:11:24,653 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22a0043c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:11:24,654 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f9d2cc3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:11:24,751 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4e5af57{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/java.io.tmpdir/jetty-localhost-40705-hadoop-hdfs-3_4_1-tests_jar-_-any-10021620966072178250/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:11:24,752 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@68aaa55{HTTP/1.1, (http/1.1)}{localhost:40705}
2024-12-16T00:11:24,752 INFO  [Time-limited test {}] server.Server(415): Started @3417ms
2024-12-16T00:11:24,755 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:11:25,547 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54/dfs/data/data1/current/BP-1593260936-172.17.0.2-1734307883021/current, will proceed with Du for space computation calculation, 
2024-12-16T00:11:25,547 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54/dfs/data/data4/current/BP-1593260936-172.17.0.2-1734307883021/current, will proceed with Du for space computation calculation, 
2024-12-16T00:11:25,547 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54/dfs/data/data3/current/BP-1593260936-172.17.0.2-1734307883021/current, will proceed with Du for space computation calculation, 
2024-12-16T00:11:25,547 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54/dfs/data/data2/current/BP-1593260936-172.17.0.2-1734307883021/current, will proceed with Du for space computation calculation, 
2024-12-16T00:11:25,581 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:11:25,581 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:11:25,625 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x560483c1cf53c013 with lease ID 0x32adcc83b8f8cb1e: Processing first storage report for DS-db6a61ad-bddf-4551-b34c-3a1062180e24 from datanode DatanodeRegistration(127.0.0.1:34417, datanodeUuid=109aa2fb-53fa-453b-9319-bb9728ab870a, infoPort=35047, infoSecurePort=0, ipcPort=41855, storageInfo=lv=-57;cid=testClusterID;nsid=68746245;c=1734307883021)
2024-12-16T00:11:25,626 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x560483c1cf53c013 with lease ID 0x32adcc83b8f8cb1e: from storage DS-db6a61ad-bddf-4551-b34c-3a1062180e24 node DatanodeRegistration(127.0.0.1:34417, datanodeUuid=109aa2fb-53fa-453b-9319-bb9728ab870a, infoPort=35047, infoSecurePort=0, ipcPort=41855, storageInfo=lv=-57;cid=testClusterID;nsid=68746245;c=1734307883021), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-16T00:11:25,627 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xab68e348638cbb56 with lease ID 0x32adcc83b8f8cb1d: Processing first storage report for DS-70a71296-0784-4395-a321-17403e706bdd from datanode DatanodeRegistration(127.0.0.1:40809, datanodeUuid=05f02631-6a58-4943-a09b-77009a366199, infoPort=43815, infoSecurePort=0, ipcPort=38143, storageInfo=lv=-57;cid=testClusterID;nsid=68746245;c=1734307883021)
2024-12-16T00:11:25,627 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xab68e348638cbb56 with lease ID 0x32adcc83b8f8cb1d: from storage DS-70a71296-0784-4395-a321-17403e706bdd node DatanodeRegistration(127.0.0.1:40809, datanodeUuid=05f02631-6a58-4943-a09b-77009a366199, infoPort=43815, infoSecurePort=0, ipcPort=38143, storageInfo=lv=-57;cid=testClusterID;nsid=68746245;c=1734307883021), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-16T00:11:25,627 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x560483c1cf53c013 with lease ID 0x32adcc83b8f8cb1e: Processing first storage report for DS-7efc5c0e-dba0-47f1-9e1f-9b22b2a8c621 from datanode DatanodeRegistration(127.0.0.1:34417, datanodeUuid=109aa2fb-53fa-453b-9319-bb9728ab870a, infoPort=35047, infoSecurePort=0, ipcPort=41855, storageInfo=lv=-57;cid=testClusterID;nsid=68746245;c=1734307883021)
2024-12-16T00:11:25,628 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x560483c1cf53c013 with lease ID 0x32adcc83b8f8cb1e: from storage DS-7efc5c0e-dba0-47f1-9e1f-9b22b2a8c621 node DatanodeRegistration(127.0.0.1:34417, datanodeUuid=109aa2fb-53fa-453b-9319-bb9728ab870a, infoPort=35047, infoSecurePort=0, ipcPort=41855, storageInfo=lv=-57;cid=testClusterID;nsid=68746245;c=1734307883021), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:11:25,628 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xab68e348638cbb56 with lease ID 0x32adcc83b8f8cb1d: Processing first storage report for DS-974be9d3-a0e5-44ae-894e-296137cbd59e from datanode DatanodeRegistration(127.0.0.1:40809, datanodeUuid=05f02631-6a58-4943-a09b-77009a366199, infoPort=43815, infoSecurePort=0, ipcPort=38143, storageInfo=lv=-57;cid=testClusterID;nsid=68746245;c=1734307883021)
2024-12-16T00:11:25,628 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xab68e348638cbb56 with lease ID 0x32adcc83b8f8cb1d: from storage DS-974be9d3-a0e5-44ae-894e-296137cbd59e node DatanodeRegistration(127.0.0.1:40809, datanodeUuid=05f02631-6a58-4943-a09b-77009a366199, infoPort=43815, infoSecurePort=0, ipcPort=38143, storageInfo=lv=-57;cid=testClusterID;nsid=68746245;c=1734307883021), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-16T00:11:25,668 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef
2024-12-16T00:11:25,728 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54/zookeeper_0, clientPort=58822, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-16T00:11:25,737 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=58822
2024-12-16T00:11:25,745 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:11:25,747 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:11:25,943 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:11:25,944 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:11:26,358 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17 with version=8
2024-12-16T00:11:26,358 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/hbase-staging
2024-12-16T00:11:26,460 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-16T00:11:26,686 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/b878af803003:0 server-side Connection retries=45
2024-12-16T00:11:26,701 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:11:26,702 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:11:26,702 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:11:26,702 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:11:26,702 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:11:26,816 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:11:26,865 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-16T00:11:26,873 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-16T00:11:26,876 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:11:26,897 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 10473 (auto-detected)
2024-12-16T00:11:26,898 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-12-16T00:11:26,914 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:44821
2024-12-16T00:11:26,920 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:11:26,922 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:11:26,934 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:44821 connecting to ZooKeeper ensemble=127.0.0.1:58822
2024-12-16T00:11:27,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:448210x0, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:11:27,049 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44821-0x1002c2d4f110000 connected
2024-12-16T00:11:27,124 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:11:27,128 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:11:27,133 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:11:27,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44821
2024-12-16T00:11:27,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44821
2024-12-16T00:11:27,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44821
2024-12-16T00:11:27,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44821
2024-12-16T00:11:27,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44821
2024-12-16T00:11:27,145 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17, hbase.cluster.distributed=false
2024-12-16T00:11:27,206 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/b878af803003:0 server-side Connection retries=45
2024-12-16T00:11:27,206 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:11:27,206 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:11:27,207 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:11:27,207 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:11:27,207 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:11:27,209 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:11:27,212 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:11:27,213 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45425
2024-12-16T00:11:27,214 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-16T00:11:27,219 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-16T00:11:27,220 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:11:27,225 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:11:27,229 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:45425 connecting to ZooKeeper ensemble=127.0.0.1:58822
2024-12-16T00:11:27,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454250x0, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:11:27,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:454250x0, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:11:27,238 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45425-0x1002c2d4f110001 connected
2024-12-16T00:11:27,239 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:11:27,240 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:11:27,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45425
2024-12-16T00:11:27,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45425
2024-12-16T00:11:27,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45425
2024-12-16T00:11:27,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45425
2024-12-16T00:11:27,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45425
2024-12-16T00:11:27,244 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/b878af803003,44821,1734307886455
2024-12-16T00:11:27,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:11:27,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:11:27,257 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b878af803003,44821,1734307886455
2024-12-16T00:11:27,258 DEBUG [M:0;b878af803003:44821 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b878af803003:44821
2024-12-16T00:11:27,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:11:27,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:11:27,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:27,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:27,280 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:11:27,281 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b878af803003,44821,1734307886455 from backup master directory
2024-12-16T00:11:27,281 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:11:27,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b878af803003,44821,1734307886455
2024-12-16T00:11:27,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:11:27,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:11:27,294 WARN  [master/b878af803003:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:11:27,294 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b878af803003,44821,1734307886455
2024-12-16T00:11:27,297 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-16T00:11:27,298 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-16T00:11:27,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:11:27,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:11:27,361 DEBUG [master/b878af803003:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/hbase.id with ID: 775e324d-dbdb-4fb9-a6cd-5f6208e16fd0
2024-12-16T00:11:27,401 INFO  [master/b878af803003:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:11:27,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:27,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:27,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:11:27,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:11:27,461 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-16T00:11:27,463 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-16T00:11:27,468 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:11:27,495 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:11:27,496 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:11:27,510 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store
2024-12-16T00:11:27,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:11:27,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:11:27,529 INFO  [master/b878af803003:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-16T00:11:27,530 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:11:27,531 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:11:27,531 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:11:27,531 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:11:27,531 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:11:27,531 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:11:27,532 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:11:27,532 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:11:27,534 WARN  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/.initializing
2024-12-16T00:11:27,534 DEBUG [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/WALs/b878af803003,44821,1734307886455
2024-12-16T00:11:27,548 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C44821%2C1734307886455, suffix=, logDir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/WALs/b878af803003,44821,1734307886455, archiveDir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/oldWALs, maxLogs=10
2024-12-16T00:11:27,555 INFO  [master/b878af803003:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C44821%2C1734307886455.1734307887554
2024-12-16T00:11:27,556 DEBUG [master/b878af803003:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(752): Using builder API via reflection for DFS file creation replicate flag.
2024-12-16T00:11:27,556 DEBUG [master/b878af803003:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(762): Using builder API via reflection for DFS file creation noLocalWrite flag.
2024-12-16T00:11:27,572 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/WALs/b878af803003,44821,1734307886455/b878af803003%2C44821%2C1734307886455.1734307887554
2024-12-16T00:11:27,579 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35047:35047),(127.0.0.1/127.0.0.1:43815:43815)]
2024-12-16T00:11:27,580 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:11:27,580 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:11:27,583 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:11:27,584 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:11:27,615 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:11:27,636 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-16T00:11:27,639 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:27,642 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:11:27,642 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:11:27,646 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-16T00:11:27,646 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:27,647 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:11:27,647 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:11:27,650 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-16T00:11:27,650 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:27,651 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:11:27,652 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:11:27,654 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-16T00:11:27,654 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:27,655 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:11:27,659 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:11:27,660 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:11:27,668 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-16T00:11:27,672 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:11:27,676 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:11:27,677 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=688226, jitterRate=-0.12487654387950897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-16T00:11:27,681 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:11:27,682 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-16T00:11:27,708 DEBUG [master/b878af803003:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9c7f3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:11:27,736 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-16T00:11:27,746 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-16T00:11:27,746 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-16T00:11:27,748 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-16T00:11:27,749 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec
2024-12-16T00:11:27,754 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec
2024-12-16T00:11:27,754 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-16T00:11:27,776 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-16T00:11:27,786 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-16T00:11:27,821 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-16T00:11:27,823 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-16T00:11:27,824 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-16T00:11:27,834 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-16T00:11:27,836 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-16T00:11:27,839 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-16T00:11:27,846 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-16T00:11:27,847 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-16T00:11:27,854 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-16T00:11:27,863 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-16T00:11:27,871 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-16T00:11:27,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:11:27,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:11:27,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:27,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:27,880 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=b878af803003,44821,1734307886455, sessionid=0x1002c2d4f110000, setting cluster-up flag (Was=false)
2024-12-16T00:11:27,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:27,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:27,929 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-16T00:11:27,931 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,44821,1734307886455
2024-12-16T00:11:27,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:27,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:27,979 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-16T00:11:27,982 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,44821,1734307886455
2024-12-16T00:11:28,055 DEBUG [RS:0;b878af803003:45425 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b878af803003:45425
2024-12-16T00:11:28,054 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-16T00:11:28,056 INFO  [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1008): ClusterId : 775e324d-dbdb-4fb9-a6cd-5f6208e16fd0
2024-12-16T00:11:28,059 DEBUG [RS:0;b878af803003:45425 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-16T00:11:28,061 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-16T00:11:28,064 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-16T00:11:28,070 DEBUG [RS:0;b878af803003:45425 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-16T00:11:28,070 DEBUG [RS:0;b878af803003:45425 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-16T00:11:28,070 DEBUG [master/b878af803003:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b878af803003,44821,1734307886455
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-16T00:11:28,073 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:11:28,074 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:11:28,074 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:11:28,074 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:11:28,074 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b878af803003:0, corePoolSize=10, maxPoolSize=10
2024-12-16T00:11:28,074 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,075 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:11:28,075 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,076 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734307918076
2024-12-16T00:11:28,077 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-16T00:11:28,078 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-16T00:11:28,079 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:11:28,079 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-16T00:11:28,080 DEBUG [RS:0;b878af803003:45425 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-16T00:11:28,081 DEBUG [RS:0;b878af803003:45425 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ba76b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:11:28,081 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-16T00:11:28,081 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-16T00:11:28,082 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-16T00:11:28,082 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-16T00:11:28,082 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:28,082 DEBUG [RS:0;b878af803003:45425 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52fbed8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:11:28,083 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-16T00:11:28,084 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:28,084 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-16T00:11:28,084 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:11:28,084 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-16T00:11:28,086 INFO  [RS:0;b878af803003:45425 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-16T00:11:28,086 INFO  [RS:0;b878af803003:45425 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-16T00:11:28,086 DEBUG [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-16T00:11:28,087 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-16T00:11:28,087 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-16T00:11:28,088 INFO  [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(3073): reportForDuty to master=b878af803003,44821,1734307886455 with isa=b878af803003/172.17.0.2:45425, startcode=1734307887205
2024-12-16T00:11:28,089 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734307888089,5,FailOnTimeoutGroup]
2024-12-16T00:11:28,089 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734307888089,5,FailOnTimeoutGroup]
2024-12-16T00:11:28,089 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:28,090 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-16T00:11:28,091 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:28,091 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:28,094 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:11:28,095 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:11:28,097 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-16T00:11:28,097 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17
2024-12-16T00:11:28,100 DEBUG [RS:0;b878af803003:45425 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:11:28,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:11:28,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:11:28,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:11:28,113 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:11:28,116 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:11:28,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:28,118 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:11:28,118 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:11:28,121 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:11:28,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:28,122 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:11:28,123 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:11:28,125 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:11:28,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:28,126 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:11:28,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740
2024-12-16T00:11:28,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740
2024-12-16T00:11:28,131 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:11:28,134 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:11:28,142 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:11:28,143 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=692901, jitterRate=-0.11893203854560852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:11:28,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:11:28,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:11:28,146 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:11:28,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:11:28,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:11:28,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:11:28,147 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-16T00:11:28,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:11:28,149 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:11:28,149 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-16T00:11:28,154 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-16T00:11:28,161 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-16T00:11:28,162 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-16T00:11:28,167 INFO  [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37181, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:11:28,172 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44821 {}] master.ServerManager(332): Checking decommissioned status of RegionServer b878af803003,45425,1734307887205
2024-12-16T00:11:28,174 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44821 {}] master.ServerManager(486): Registering regionserver=b878af803003,45425,1734307887205
2024-12-16T00:11:28,185 DEBUG [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17
2024-12-16T00:11:28,185 DEBUG [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:33445
2024-12-16T00:11:28,185 DEBUG [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-16T00:11:28,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:11:28,196 DEBUG [RS:0;b878af803003:45425 {}] zookeeper.ZKUtil(111): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b878af803003,45425,1734307887205
2024-12-16T00:11:28,197 WARN  [RS:0;b878af803003:45425 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:11:28,197 INFO  [RS:0;b878af803003:45425 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:11:28,197 DEBUG [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205
2024-12-16T00:11:28,199 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b878af803003,45425,1734307887205]
2024-12-16T00:11:28,213 DEBUG [RS:0;b878af803003:45425 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-16T00:11:28,223 INFO  [RS:0;b878af803003:45425 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-16T00:11:28,233 INFO  [RS:0;b878af803003:45425 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-16T00:11:28,235 INFO  [RS:0;b878af803003:45425 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-16T00:11:28,235 INFO  [RS:0;b878af803003:45425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:28,236 INFO  [RS:0;b878af803003:45425 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-16T00:11:28,241 INFO  [RS:0;b878af803003:45425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:28,241 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,241 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,241 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,242 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,242 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,242 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:11:28,242 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,242 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,242 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,242 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,242 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:11:28,242 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:11:28,243 DEBUG [RS:0;b878af803003:45425 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:11:28,243 INFO  [RS:0;b878af803003:45425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:28,243 INFO  [RS:0;b878af803003:45425 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:28,243 INFO  [RS:0;b878af803003:45425 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:28,244 INFO  [RS:0;b878af803003:45425 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:28,244 INFO  [RS:0;b878af803003:45425 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,45425,1734307887205-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:11:28,259 INFO  [RS:0;b878af803003:45425 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-16T00:11:28,260 INFO  [RS:0;b878af803003:45425 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,45425,1734307887205-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:28,275 INFO  [RS:0;b878af803003:45425 {}] regionserver.Replication(204): b878af803003,45425,1734307887205 started
2024-12-16T00:11:28,275 INFO  [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1767): Serving as b878af803003,45425,1734307887205, RpcServer on b878af803003/172.17.0.2:45425, sessionid=0x1002c2d4f110001
2024-12-16T00:11:28,276 DEBUG [RS:0;b878af803003:45425 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-16T00:11:28,276 DEBUG [RS:0;b878af803003:45425 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b878af803003,45425,1734307887205
2024-12-16T00:11:28,276 DEBUG [RS:0;b878af803003:45425 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,45425,1734307887205'
2024-12-16T00:11:28,276 DEBUG [RS:0;b878af803003:45425 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-16T00:11:28,277 DEBUG [RS:0;b878af803003:45425 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-16T00:11:28,278 DEBUG [RS:0;b878af803003:45425 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-16T00:11:28,278 DEBUG [RS:0;b878af803003:45425 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-16T00:11:28,278 DEBUG [RS:0;b878af803003:45425 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b878af803003,45425,1734307887205
2024-12-16T00:11:28,278 DEBUG [RS:0;b878af803003:45425 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,45425,1734307887205'
2024-12-16T00:11:28,278 DEBUG [RS:0;b878af803003:45425 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-16T00:11:28,279 DEBUG [RS:0;b878af803003:45425 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-16T00:11:28,279 DEBUG [RS:0;b878af803003:45425 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-16T00:11:28,279 INFO  [RS:0;b878af803003:45425 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-16T00:11:28,279 INFO  [RS:0;b878af803003:45425 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-16T00:11:28,313 WARN  [b878af803003:44821 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-16T00:11:28,391 INFO  [RS:0;b878af803003:45425 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C45425%2C1734307887205, suffix=, logDir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205, archiveDir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/oldWALs, maxLogs=32
2024-12-16T00:11:28,394 INFO  [RS:0;b878af803003:45425 {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45425%2C1734307887205.1734307888393
2024-12-16T00:11:28,401 INFO  [RS:0;b878af803003:45425 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307888393
2024-12-16T00:11:28,401 DEBUG [RS:0;b878af803003:45425 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35047:35047),(127.0.0.1/127.0.0.1:43815:43815)]
2024-12-16T00:11:28,566 DEBUG [b878af803003:44821 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-16T00:11:28,575 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b878af803003,45425,1734307887205
2024-12-16T00:11:28,580 INFO  [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,45425,1734307887205, state=OPENING
2024-12-16T00:11:28,601 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-16T00:11:28,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:28,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:28,610 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:11:28,610 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:11:28,612 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=b878af803003,45425,1734307887205}]
2024-12-16T00:11:28,785 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to b878af803003,45425,1734307887205
2024-12-16T00:11:28,786 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-16T00:11:28,789 INFO  [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52910, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-16T00:11:28,799 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-16T00:11:28,800 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:11:28,803 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C45425%2C1734307887205.meta, suffix=.meta, logDir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205, archiveDir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/oldWALs, maxLogs=32
2024-12-16T00:11:28,805 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45425%2C1734307887205.meta.1734307888805.meta
2024-12-16T00:11:28,813 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.meta.1734307888805.meta
2024-12-16T00:11:28,813 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35047:35047),(127.0.0.1/127.0.0.1:43815:43815)]
2024-12-16T00:11:28,814 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:11:28,815 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-16T00:11:28,864 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-16T00:11:28,868 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-16T00:11:28,871 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-16T00:11:28,871 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:11:28,872 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-16T00:11:28,872 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-16T00:11:28,874 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:11:28,876 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:11:28,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:28,877 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:11:28,877 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:11:28,878 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:11:28,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:28,879 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:11:28,880 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:11:28,881 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:11:28,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:28,882 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:11:28,883 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740
2024-12-16T00:11:28,886 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740
2024-12-16T00:11:28,888 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:11:28,891 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:11:28,892 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=850884, jitterRate=0.0819559246301651}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:11:28,894 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:11:28,900 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734307888779
2024-12-16T00:11:28,909 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-16T00:11:28,910 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-16T00:11:28,910 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,45425,1734307887205
2024-12-16T00:11:28,912 INFO  [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,45425,1734307887205, state=OPEN
2024-12-16T00:11:28,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:11:28,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:11:28,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:11:28,953 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:11:28,958 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-16T00:11:28,959 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=b878af803003,45425,1734307887205 in 340 msec
2024-12-16T00:11:28,966 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-16T00:11:28,966 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 806 msec
2024-12-16T00:11:28,973 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 953 msec
2024-12-16T00:11:28,973 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734307888973, completionTime=-1
2024-12-16T00:11:28,973 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-16T00:11:28,973 DEBUG [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-16T00:11:29,004 DEBUG [hconnection-0x397eff2f-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:11:29,006 INFO  [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52926, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:11:29,014 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-16T00:11:29,014 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734307949014
2024-12-16T00:11:29,014 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734308009014
2024-12-16T00:11:29,014 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 41 msec
2024-12-16T00:11:29,062 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44821,1734307886455-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:29,062 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44821,1734307886455-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:29,062 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44821,1734307886455-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:29,064 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b878af803003:44821, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:29,064 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:29,069 DEBUG [master/b878af803003:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-16T00:11:29,071 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-16T00:11:29,073 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:11:29,078 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-16T00:11:29,081 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-16T00:11:29,082 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:29,084 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-16T00:11:29,095 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:11:29,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:11:29,099 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => cb2fa9e4995d7f54e1d8bed7be70c3c1, NAME => 'hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17
2024-12-16T00:11:29,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:11:29,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:11:29,110 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:11:29,110 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing cb2fa9e4995d7f54e1d8bed7be70c3c1, disabling compactions & flushes
2024-12-16T00:11:29,110 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:11:29,110 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:11:29,110 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1. after waiting 0 ms
2024-12-16T00:11:29,110 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:11:29,111 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:11:29,111 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for cb2fa9e4995d7f54e1d8bed7be70c3c1:

2024-12-16T00:11:29,113 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-16T00:11:29,118 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734307889113"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734307889113"}]},"ts":"1734307889113"}
2024-12-16T00:11:29,138 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-16T00:11:29,140 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-16T00:11:29,142 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734307889140"}]},"ts":"1734307889140"}
2024-12-16T00:11:29,147 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-16T00:11:29,164 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=cb2fa9e4995d7f54e1d8bed7be70c3c1, ASSIGN}]
2024-12-16T00:11:29,166 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=cb2fa9e4995d7f54e1d8bed7be70c3c1, ASSIGN
2024-12-16T00:11:29,168 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=cb2fa9e4995d7f54e1d8bed7be70c3c1, ASSIGN; state=OFFLINE, location=b878af803003,45425,1734307887205; forceNewPlan=false, retain=false
2024-12-16T00:11:29,319 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=cb2fa9e4995d7f54e1d8bed7be70c3c1, regionState=OPENING, regionLocation=b878af803003,45425,1734307887205
2024-12-16T00:11:29,326 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure cb2fa9e4995d7f54e1d8bed7be70c3c1, server=b878af803003,45425,1734307887205}]
2024-12-16T00:11:29,482 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to b878af803003,45425,1734307887205
2024-12-16T00:11:29,489 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:11:29,490 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => cb2fa9e4995d7f54e1d8bed7be70c3c1, NAME => 'hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:11:29,490 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace cb2fa9e4995d7f54e1d8bed7be70c3c1
2024-12-16T00:11:29,490 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:11:29,491 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for cb2fa9e4995d7f54e1d8bed7be70c3c1
2024-12-16T00:11:29,491 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for cb2fa9e4995d7f54e1d8bed7be70c3c1
2024-12-16T00:11:29,494 INFO  [StoreOpener-cb2fa9e4995d7f54e1d8bed7be70c3c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cb2fa9e4995d7f54e1d8bed7be70c3c1 
2024-12-16T00:11:29,497 INFO  [StoreOpener-cb2fa9e4995d7f54e1d8bed7be70c3c1-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb2fa9e4995d7f54e1d8bed7be70c3c1 columnFamilyName info
2024-12-16T00:11:29,497 DEBUG [StoreOpener-cb2fa9e4995d7f54e1d8bed7be70c3c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:29,498 INFO  [StoreOpener-cb2fa9e4995d7f54e1d8bed7be70c3c1-1 {}] regionserver.HStore(327): Store=cb2fa9e4995d7f54e1d8bed7be70c3c1/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:11:29,500 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/namespace/cb2fa9e4995d7f54e1d8bed7be70c3c1
2024-12-16T00:11:29,501 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/namespace/cb2fa9e4995d7f54e1d8bed7be70c3c1
2024-12-16T00:11:29,505 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for cb2fa9e4995d7f54e1d8bed7be70c3c1
2024-12-16T00:11:29,509 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/namespace/cb2fa9e4995d7f54e1d8bed7be70c3c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:11:29,510 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened cb2fa9e4995d7f54e1d8bed7be70c3c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=789622, jitterRate=0.0040566325187683105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:11:29,511 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for cb2fa9e4995d7f54e1d8bed7be70c3c1:

2024-12-16T00:11:29,513 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1., pid=6, masterSystemTime=1734307889481
2024-12-16T00:11:29,517 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:11:29,517 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:11:29,518 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=cb2fa9e4995d7f54e1d8bed7be70c3c1, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,45425,1734307887205
2024-12-16T00:11:29,526 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-16T00:11:29,527 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure cb2fa9e4995d7f54e1d8bed7be70c3c1, server=b878af803003,45425,1734307887205 in 195 msec
2024-12-16T00:11:29,530 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-16T00:11:29,530 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=cb2fa9e4995d7f54e1d8bed7be70c3c1, ASSIGN in 363 msec
2024-12-16T00:11:29,531 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-16T00:11:29,532 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734307889531"}]},"ts":"1734307889531"}
2024-12-16T00:11:29,534 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-16T00:11:29,556 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-16T00:11:29,561 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 483 msec
2024-12-16T00:11:29,582 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-16T00:11:29,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:11:29,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:29,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:11:29,651 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-16T00:11:29,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:11:29,685 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 34 msec
2024-12-16T00:11:29,697 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-16T00:11:29,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:11:29,734 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 36 msec
2024-12-16T00:11:29,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-16T00:11:29,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-16T00:11:29,776 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 2.482sec
2024-12-16T00:11:29,778 INFO  [master/b878af803003:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-16T00:11:29,780 INFO  [master/b878af803003:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-16T00:11:29,781 INFO  [master/b878af803003:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-16T00:11:29,782 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-16T00:11:29,782 INFO  [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-16T00:11:29,784 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44821,1734307886455-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:11:29,785 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44821,1734307886455-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-16T00:11:29,793 DEBUG [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-16T00:11:29,794 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-16T00:11:29,794 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44821,1734307886455-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:11:29,862 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1bfed419 to 127.0.0.1:58822 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6af8af4e
2024-12-16T00:11:29,863 WARN  [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry
2024-12-16T00:11:29,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7650ec16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:11:29,880 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-16T00:11:29,880 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-16T00:11:29,889 DEBUG [hconnection-0x4f6c544d-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:11:29,916 INFO  [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44536, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:11:29,927 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=b878af803003,44821,1734307886455
2024-12-16T00:11:29,927 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:11:29,933 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-16T00:11:29,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-16T00:11:29,941 INFO  [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42202, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-16T00:11:29,946 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44821 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-16T00:11:29,946 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44821 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-16T00:11:29,949 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44821 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-16T00:11:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44821 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-12-16T00:11:29,952 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-16T00:11:29,953 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:29,955 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-16T00:11:29,955 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44821 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 9
2024-12-16T00:11:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44821 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-16T00:11:29,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741837_1013 (size=389)
2024-12-16T00:11:29,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741837_1013 (size=389)
2024-12-16T00:11:29,968 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 64e3193e86d9d4ff92f22ce4aacde553, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17
2024-12-16T00:11:29,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741838_1014 (size=72)
2024-12-16T00:11:29,977 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741838_1014 (size=72)
2024-12-16T00:11:29,978 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:11:29,978 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing 64e3193e86d9d4ff92f22ce4aacde553, disabling compactions & flushes
2024-12-16T00:11:29,978 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:11:29,978 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:11:29,978 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553. after waiting 0 ms
2024-12-16T00:11:29,978 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:11:29,978 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:11:29,978 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 64e3193e86d9d4ff92f22ce4aacde553:

2024-12-16T00:11:29,980 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-16T00:11:29,981 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1734307889980"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734307889980"}]},"ts":"1734307889980"}
2024-12-16T00:11:29,983 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-16T00:11:29,985 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-16T00:11:29,985 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734307889985"}]},"ts":"1734307889985"}
2024-12-16T00:11:29,988 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-12-16T00:11:30,004 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=64e3193e86d9d4ff92f22ce4aacde553, ASSIGN}]
2024-12-16T00:11:30,006 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=64e3193e86d9d4ff92f22ce4aacde553, ASSIGN
2024-12-16T00:11:30,008 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=64e3193e86d9d4ff92f22ce4aacde553, ASSIGN; state=OFFLINE, location=b878af803003,45425,1734307887205; forceNewPlan=false, retain=false
2024-12-16T00:11:30,159 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=64e3193e86d9d4ff92f22ce4aacde553, regionState=OPENING, regionLocation=b878af803003,45425,1734307887205
2024-12-16T00:11:30,167 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 64e3193e86d9d4ff92f22ce4aacde553, server=b878af803003,45425,1734307887205}]
2024-12-16T00:11:30,324 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to b878af803003,45425,1734307887205
2024-12-16T00:11:30,333 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:11:30,333 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 64e3193e86d9d4ff92f22ce4aacde553, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:11:30,334 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 64e3193e86d9d4ff92f22ce4aacde553
2024-12-16T00:11:30,334 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:11:30,334 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 64e3193e86d9d4ff92f22ce4aacde553
2024-12-16T00:11:30,334 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 64e3193e86d9d4ff92f22ce4aacde553
2024-12-16T00:11:30,336 INFO  [StoreOpener-64e3193e86d9d4ff92f22ce4aacde553-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 64e3193e86d9d4ff92f22ce4aacde553 
2024-12-16T00:11:30,338 INFO  [StoreOpener-64e3193e86d9d4ff92f22ce4aacde553-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64e3193e86d9d4ff92f22ce4aacde553 columnFamilyName info
2024-12-16T00:11:30,338 DEBUG [StoreOpener-64e3193e86d9d4ff92f22ce4aacde553-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:11:30,339 INFO  [StoreOpener-64e3193e86d9d4ff92f22ce4aacde553-1 {}] regionserver.HStore(327): Store=64e3193e86d9d4ff92f22ce4aacde553/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:11:30,341 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553
2024-12-16T00:11:30,341 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553
2024-12-16T00:11:30,344 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 64e3193e86d9d4ff92f22ce4aacde553
2024-12-16T00:11:30,347 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:11:30,348 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 64e3193e86d9d4ff92f22ce4aacde553; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=742317, jitterRate=-0.056095466017723083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:11:30,349 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 64e3193e86d9d4ff92f22ce4aacde553:

2024-12-16T00:11:30,350 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553., pid=11, masterSystemTime=1734307890324
2024-12-16T00:11:30,353 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:11:30,353 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:11:30,354 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=64e3193e86d9d4ff92f22ce4aacde553, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,45425,1734307887205
2024-12-16T00:11:30,360 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-16T00:11:30,361 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 64e3193e86d9d4ff92f22ce4aacde553, server=b878af803003,45425,1734307887205 in 190 msec
2024-12-16T00:11:30,363 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-16T00:11:30,363 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=64e3193e86d9d4ff92f22ce4aacde553, ASSIGN in 356 msec
2024-12-16T00:11:30,364 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-16T00:11:30,365 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734307890364"}]},"ts":"1734307890364"}
2024-12-16T00:11:30,367 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-12-16T00:11:30,424 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-16T00:11:30,430 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 475 msec
2024-12-16T00:11:34,380 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-16T00:11:34,419 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-16T00:11:34,421 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-16T00:11:34,421 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-12-16T00:11:36,863 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-16T00:11:36,865 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-16T00:11:36,867 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-16T00:11:36,867 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-16T00:11:36,868 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace
2024-12-16T00:11:36,868 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer
2024-12-16T00:11:36,869 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-16T00:11:36,869 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-16T00:11:36,870 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-16T00:11:36,870 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-16T00:11:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44821 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-16T00:11:39,971 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling, procId: 9 completed
2024-12-16T00:11:39,977 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-12-16T00:11:39,978 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:11:39,979 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45425%2C1734307887205.1734307899978
2024-12-16T00:11:39,989 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307888393 with entries=4, filesize=947 B; new WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307899978
2024-12-16T00:11:39,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35047:35047),(127.0.0.1/127.0.0.1:43815:43815)]
2024-12-16T00:11:39,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307888393 is not closed yet, will try archiving it next time
2024-12-16T00:11:39,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741833_1009 (size=955)
2024-12-16T00:11:39,993 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741833_1009 (size=955)
2024-12-16T00:11:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45425 {}] regionserver.HRegion(8581): Flush requested on 64e3193e86d9d4ff92f22ce4aacde553
2024-12-16T00:11:52,061 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 64e3193e86d9d4ff92f22ce4aacde553 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-16T00:11:52,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/63bbc235a5784cad975740fd7dc1d215 is 1080, key is row0001/info:/1734307899995/Put/seqid=0
2024-12-16T00:11:52,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741840_1016 (size=12509)
2024-12-16T00:11:52,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741840_1016 (size=12509)
2024-12-16T00:11:52,129 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/63bbc235a5784cad975740fd7dc1d215
2024-12-16T00:11:52,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/63bbc235a5784cad975740fd7dc1d215 as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/63bbc235a5784cad975740fd7dc1d215
2024-12-16T00:11:52,189 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/63bbc235a5784cad975740fd7dc1d215, entries=7, sequenceid=11, filesize=12.2 K
2024-12-16T00:11:52,192 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 64e3193e86d9d4ff92f22ce4aacde553 in 131ms, sequenceid=11, compaction requested=false
2024-12-16T00:11:52,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 64e3193e86d9d4ff92f22ce4aacde553:

2024-12-16T00:11:55,664 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-16T00:11:59,085 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:11:59,089 INFO  [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43626, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:12:00,081 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45425%2C1734307887205.1734307920081
2024-12-16T00:12:00,299 INFO  [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 213 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:00,301 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307899978 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307920081
2024-12-16T00:12:00,301 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35047:35047)]
2024-12-16T00:12:00,301 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307899978 is not closed yet, will try archiving it next time
2024-12-16T00:12:00,305 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741839_1015 (size=12399)
2024-12-16T00:12:00,306 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741839_1015 (size=12399)
2024-12-16T00:12:00,506 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:02,713 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:04,917 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:07,121 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45425 {}] regionserver.HRegion(8581): Flush requested on 64e3193e86d9d4ff92f22ce4aacde553
2024-12-16T00:12:07,121 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 64e3193e86d9d4ff92f22ce4aacde553 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-16T00:12:07,324 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:07,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/55765891fb584f2c8f37fef15c9c645b is 1080, key is row0008/info:/1734307914063/Put/seqid=0
2024-12-16T00:12:07,343 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741842_1018 (size=12509)
2024-12-16T00:12:07,344 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741842_1018 (size=12509)
2024-12-16T00:12:07,344 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/55765891fb584f2c8f37fef15c9c645b
2024-12-16T00:12:07,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/55765891fb584f2c8f37fef15c9c645b as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/55765891fb584f2c8f37fef15c9c645b
2024-12-16T00:12:07,363 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/55765891fb584f2c8f37fef15c9c645b, entries=7, sequenceid=21, filesize=12.2 K
2024-12-16T00:12:07,565 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:07,566 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 64e3193e86d9d4ff92f22ce4aacde553 in 444ms, sequenceid=21, compaction requested=false
2024-12-16T00:12:07,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 64e3193e86d9d4ff92f22ce4aacde553:

2024-12-16T00:12:07,566 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=24.4 K, sizeToCheck=16.0 K
2024-12-16T00:12:07,566 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:12:07,568 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/63bbc235a5784cad975740fd7dc1d215 because midkey is the same as first or last row
2024-12-16T00:12:09,378 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 252 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:10,200 INFO  [master/b878af803003:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-16T00:12:10,200 INFO  [master/b878af803003:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-16T00:12:11,585 WARN  [sync.1 {}] wal.AbstractFSWAL(1346): Requesting log roll because we exceeded slow sync threshold; count=7, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:11,589 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C45425%2C1734307887205:(num 1734307920081) roll requested
2024-12-16T00:12:11,589 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:11,590 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45425%2C1734307887205.1734307931589
2024-12-16T00:12:11,803 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 210 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:12,004 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:12,006 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307920081 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307931589
2024-12-16T00:12:12,006 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35047:35047)]
2024-12-16T00:12:12,007 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307920081 is not closed yet, will try archiving it next time
2024-12-16T00:12:12,009 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307899978 to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/oldWALs/b878af803003%2C45425%2C1734307887205.1734307899978
2024-12-16T00:12:12,012 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741841_1017 (size=7739)
2024-12-16T00:12:12,013 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741841_1017 (size=7739)
2024-12-16T00:12:13,793 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:15,334 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 64e3193e86d9d4ff92f22ce4aacde553, had cached 0 bytes from a total of 25018
2024-12-16T00:12:15,999 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:18,203 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:20,406 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:22,411 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-16T00:12:22,412 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45425%2C1734307887205.1734307942411
2024-12-16T00:12:25,664 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-16T00:12:27,428 INFO  [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 5012 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:27,428 WARN  [Time-limited test {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5012 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:27,428 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C45425%2C1734307887205:(num 1734307942411) roll requested
2024-12-16T00:12:29,395 DEBUG [master/b878af803003:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region cb2fa9e4995d7f54e1d8bed7be70c3c1 changed from -1.0 to 0.0, refreshing cache
2024-12-16T00:12:32,429 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:32,429 WARN  [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:32,431 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307931589 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307942411
2024-12-16T00:12:32,431 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35047:35047),(127.0.0.1/127.0.0.1:43815:43815)]
2024-12-16T00:12:32,431 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307931589 is not closed yet, will try archiving it next time
2024-12-16T00:12:32,433 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45425%2C1734307887205.1734307952432
2024-12-16T00:12:32,479 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741843_1019 (size=4753)
2024-12-16T00:12:32,481 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741843_1019 (size=4753)
2024-12-16T00:12:37,438 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:37,439 WARN  [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45425 {}] regionserver.HRegion(8581): Flush requested on 64e3193e86d9d4ff92f22ce4aacde553
2024-12-16T00:12:37,440 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 64e3193e86d9d4ff92f22ce4aacde553 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-16T00:12:37,492 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5012 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:37,492 WARN  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5012 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:39,441 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-16T00:12:42,443 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:42,444 WARN  [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:42,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/a98f4cae3a164d7badbe12fad1e08b45 is 1080, key is row0015/info:/1734307929124/Put/seqid=0
2024-12-16T00:12:42,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741846_1022 (size=12509)
2024-12-16T00:12:42,461 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741846_1022 (size=12509)
2024-12-16T00:12:42,461 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/a98f4cae3a164d7badbe12fad1e08b45
2024-12-16T00:12:42,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/a98f4cae3a164d7badbe12fad1e08b45 as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/a98f4cae3a164d7badbe12fad1e08b45
2024-12-16T00:12:42,481 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/a98f4cae3a164d7badbe12fad1e08b45, entries=7, sequenceid=31, filesize=12.2 K
2024-12-16T00:12:42,492 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:42,492 WARN  [sync.0 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:42,493 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307942411 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307952432
2024-12-16T00:12:42,493 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35047:35047)]
2024-12-16T00:12:42,493 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307942411 is not closed yet, will try archiving it next time
2024-12-16T00:12:42,493 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C45425%2C1734307887205:(num 1734307952432) roll requested
2024-12-16T00:12:42,493 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45425%2C1734307887205.1734307962493
2024-12-16T00:12:42,495 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741844_1020 (size=1569)
2024-12-16T00:12:42,495 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741844_1020 (size=1569)
2024-12-16T00:12:47,520 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5026 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:47,521 WARN  [sync.1 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5026 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:47,521 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 64e3193e86d9d4ff92f22ce4aacde553 in 10081ms, sequenceid=31, compaction requested=true
2024-12-16T00:12:47,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 64e3193e86d9d4ff92f22ce4aacde553:

2024-12-16T00:12:47,522 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=36.6 K, sizeToCheck=16.0 K
2024-12-16T00:12:47,522 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:12:47,522 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/63bbc235a5784cad975740fd7dc1d215 because midkey is the same as first or last row
2024-12-16T00:12:47,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64e3193e86d9d4ff92f22ce4aacde553:info, priority=-2147483648, current under compaction store size is 1
2024-12-16T00:12:47,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:12:47,525 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-16T00:12:47,528 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5032 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:47,528 WARN  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5032 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:47,529 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-16T00:12:47,531 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.HStore(1540): 64e3193e86d9d4ff92f22ce4aacde553/info is initiating minor compaction (all files)
2024-12-16T00:12:47,531 INFO  [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 64e3193e86d9d4ff92f22ce4aacde553/info in TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:12:47,531 INFO  [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/63bbc235a5784cad975740fd7dc1d215, hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/55765891fb584f2c8f37fef15c9c645b, hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/a98f4cae3a164d7badbe12fad1e08b45] into tmpdir=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp, totalSize=36.6 K
2024-12-16T00:12:47,533 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] compactions.Compactor(224): Compacting 63bbc235a5784cad975740fd7dc1d215, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1734307899995
2024-12-16T00:12:47,534 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] compactions.Compactor(224): Compacting 55765891fb584f2c8f37fef15c9c645b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1734307914063
2024-12-16T00:12:47,534 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] compactions.Compactor(224): Compacting a98f4cae3a164d7badbe12fad1e08b45, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1734307929124
2024-12-16T00:12:47,558 INFO  [RS:0;b878af803003:45425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64e3193e86d9d4ff92f22ce4aacde553#info#compaction#3 average throughput is 21.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:12:47,558 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/1930162aa0784408b567a0fe627bae70 is 1080, key is row0001/info:/1734307899995/Put/seqid=0
2024-12-16T00:12:47,564 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741848_1024 (size=27710)
2024-12-16T00:12:47,565 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741848_1024 (size=27710)
2024-12-16T00:12:47,574 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/1930162aa0784408b567a0fe627bae70 as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/1930162aa0784408b567a0fe627bae70
2024-12-16T00:12:52,529 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:52,529 WARN  [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK], DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK]]
2024-12-16T00:12:52,530 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307952432 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307962493
2024-12-16T00:12:52,531 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35047:35047),(127.0.0.1/127.0.0.1:43815:43815)]
2024-12-16T00:12:52,531 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307952432 is not closed yet, will try archiving it next time
2024-12-16T00:12:52,532 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307920081 to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/oldWALs/b878af803003%2C45425%2C1734307887205.1734307920081
2024-12-16T00:12:52,532 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C45425%2C1734307887205:(num 1734307972532) roll requested
2024-12-16T00:12:52,532 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45425%2C1734307887205.1734307972532
2024-12-16T00:12:52,555 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307931589 to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/oldWALs/b878af803003%2C45425%2C1734307887205.1734307931589
2024-12-16T00:12:52,557 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741845_1021 (size=438)
2024-12-16T00:12:52,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741845_1021 (size=438)
2024-12-16T00:12:52,559 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307942411 to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/oldWALs/b878af803003%2C45425%2C1734307887205.1734307942411
2024-12-16T00:12:52,561 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307952432 to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/oldWALs/b878af803003%2C45425%2C1734307887205.1734307952432
2024-12-16T00:12:55,664 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-16T00:12:57,534 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:57,534 WARN  [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:57,538 INFO  [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 64e3193e86d9d4ff92f22ce4aacde553/info of 64e3193e86d9d4ff92f22ce4aacde553 into 1930162aa0784408b567a0fe627bae70(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 10sec to execute.
2024-12-16T00:12:57,539 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 64e3193e86d9d4ff92f22ce4aacde553:

2024-12-16T00:12:57,539 INFO  [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553., storeName=64e3193e86d9d4ff92f22ce4aacde553/info, priority=13, startTime=1734307967524; duration=10sec
2024-12-16T00:12:57,540 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=27.1 K, sizeToCheck=16.0 K
2024-12-16T00:12:57,541 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:12:57,541 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/1930162aa0784408b567a0fe627bae70 because midkey is the same as first or last row
2024-12-16T00:12:57,541 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:12:57,541 DEBUG [RS:0;b878af803003:45425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64e3193e86d9d4ff92f22ce4aacde553:info
2024-12-16T00:12:57,566 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:57,566 WARN  [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34417,DS-db6a61ad-bddf-4551-b34c-3a1062180e24,DISK], DatanodeInfoWithStorage[127.0.0.1:40809,DS-70a71296-0784-4395-a321-17403e706bdd,DISK]]
2024-12-16T00:12:57,566 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307962493 with entries=1, filesize=531 B; new WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307972532
2024-12-16T00:12:57,566 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43815:43815),(127.0.0.1/127.0.0.1:35047:35047)]
2024-12-16T00:12:57,566 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307962493 is not closed yet, will try archiving it next time
2024-12-16T00:12:57,567 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45425%2C1734307887205.1734307977567
2024-12-16T00:12:57,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741847_1023 (size=539)
2024-12-16T00:12:57,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741847_1023 (size=539)
2024-12-16T00:12:57,591 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307962493 to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/oldWALs/b878af803003%2C45425%2C1734307887205.1734307962493
2024-12-16T00:12:57,596 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307972532 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307977567
2024-12-16T00:12:57,597 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35047:35047),(127.0.0.1/127.0.0.1:43815:43815)]
2024-12-16T00:12:57,597 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307972532 is not closed yet, will try archiving it next time
2024-12-16T00:12:57,597 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C45425%2C1734307887205:(num 1734307977567) roll requested
2024-12-16T00:12:57,597 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45425%2C1734307887205.1734307977597
2024-12-16T00:12:57,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741849_1025 (size=1258)
2024-12-16T00:12:57,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741849_1025 (size=1258)
2024-12-16T00:12:57,606 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307977567 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307977597
2024-12-16T00:12:57,606 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35047:35047),(127.0.0.1/127.0.0.1:43815:43815)]
2024-12-16T00:12:57,606 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307977567 is not closed yet, will try archiving it next time
2024-12-16T00:12:57,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741850_1026 (size=93)
2024-12-16T00:12:57,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741850_1026 (size=93)
2024-12-16T00:12:57,609 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205/b878af803003%2C45425%2C1734307887205.1734307977567 to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/oldWALs/b878af803003%2C45425%2C1734307887205.1734307977567
2024-12-16T00:13:00,335 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 64e3193e86d9d4ff92f22ce4aacde553, had cached 0 bytes from a total of 27710
2024-12-16T00:13:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45425 {}] regionserver.HRegion(8581): Flush requested on 64e3193e86d9d4ff92f22ce4aacde553
2024-12-16T00:13:09,626 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 64e3193e86d9d4ff92f22ce4aacde553 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-16T00:13:09,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/517719ba3a3a4736af5568ed12210e6c is 1080, key is row0022/info:/1734307977568/Put/seqid=0
2024-12-16T00:13:09,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741852_1028 (size=12509)
2024-12-16T00:13:09,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741852_1028 (size=12509)
2024-12-16T00:13:09,639 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/517719ba3a3a4736af5568ed12210e6c
2024-12-16T00:13:09,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/517719ba3a3a4736af5568ed12210e6c as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/517719ba3a3a4736af5568ed12210e6c
2024-12-16T00:13:09,656 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/517719ba3a3a4736af5568ed12210e6c, entries=7, sequenceid=42, filesize=12.2 K
2024-12-16T00:13:09,657 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 64e3193e86d9d4ff92f22ce4aacde553 in 32ms, sequenceid=42, compaction requested=false
2024-12-16T00:13:09,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 64e3193e86d9d4ff92f22ce4aacde553:

2024-12-16T00:13:09,658 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=39.3 K, sizeToCheck=16.0 K
2024-12-16T00:13:09,658 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:13:09,658 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/1930162aa0784408b567a0fe627bae70 because midkey is the same as first or last row
2024-12-16T00:13:17,645 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-16T00:13:17,646 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-16T00:13:17,646 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1bfed419 to 127.0.0.1:58822
2024-12-16T00:13:17,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:13:17,649 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-16T00:13:17,649 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1803308537, stopped=false
2024-12-16T00:13:17,650 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=b878af803003,44821,1734307886455
2024-12-16T00:13:17,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:13:17,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:13:17,697 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-16T00:13:17,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:17,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:17,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:13:17,698 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,45425,1734307887205' *****
2024-12-16T00:13:17,699 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-16T00:13:17,699 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:13:17,700 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:13:17,700 INFO  [RS:0;b878af803003:45425 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-16T00:13:17,700 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-16T00:13:17,700 INFO  [RS:0;b878af803003:45425 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-16T00:13:17,701 INFO  [RS:0;b878af803003:45425 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-16T00:13:17,701 INFO  [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(3579): Received CLOSE for cb2fa9e4995d7f54e1d8bed7be70c3c1
2024-12-16T00:13:17,702 INFO  [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(3579): Received CLOSE for 64e3193e86d9d4ff92f22ce4aacde553
2024-12-16T00:13:17,702 INFO  [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1224): stopping server b878af803003,45425,1734307887205
2024-12-16T00:13:17,702 DEBUG [RS:0;b878af803003:45425 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:13:17,702 INFO  [RS:0;b878af803003:45425 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-16T00:13:17,702 INFO  [RS:0;b878af803003:45425 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-16T00:13:17,702 INFO  [RS:0;b878af803003:45425 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-16T00:13:17,702 INFO  [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-16T00:13:17,702 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing cb2fa9e4995d7f54e1d8bed7be70c3c1, disabling compactions & flushes
2024-12-16T00:13:17,703 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:13:17,703 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:13:17,703 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1. after waiting 0 ms
2024-12-16T00:13:17,703 INFO  [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-16T00:13:17,703 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:13:17,703 DEBUG [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, cb2fa9e4995d7f54e1d8bed7be70c3c1=hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1., 64e3193e86d9d4ff92f22ce4aacde553=TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.}
2024-12-16T00:13:17,703 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing cb2fa9e4995d7f54e1d8bed7be70c3c1 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-16T00:13:17,703 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:13:17,703 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:13:17,703 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:13:17,703 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:13:17,704 DEBUG [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 64e3193e86d9d4ff92f22ce4aacde553, cb2fa9e4995d7f54e1d8bed7be70c3c1
2024-12-16T00:13:17,703 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:13:17,704 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.81 KB heapSize=5.32 KB
2024-12-16T00:13:17,719 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/namespace/cb2fa9e4995d7f54e1d8bed7be70c3c1/.tmp/info/69796aab5be049cc9e26f4662f8451b2 is 45, key is default/info:d/1734307889661/Put/seqid=0
2024-12-16T00:13:17,721 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/.tmp/info/49fcd85b62da41699c063aaacdd306d3 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553./info:regioninfo/1734307890354/Put/seqid=0
2024-12-16T00:13:17,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741853_1029 (size=5037)
2024-12-16T00:13:17,726 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741853_1029 (size=5037)
2024-12-16T00:13:17,726 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/namespace/cb2fa9e4995d7f54e1d8bed7be70c3c1/.tmp/info/69796aab5be049cc9e26f4662f8451b2
2024-12-16T00:13:17,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741854_1030 (size=8172)
2024-12-16T00:13:17,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741854_1030 (size=8172)
2024-12-16T00:13:17,735 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.59 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/.tmp/info/49fcd85b62da41699c063aaacdd306d3
2024-12-16T00:13:17,735 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/namespace/cb2fa9e4995d7f54e1d8bed7be70c3c1/.tmp/info/69796aab5be049cc9e26f4662f8451b2 as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/namespace/cb2fa9e4995d7f54e1d8bed7be70c3c1/info/69796aab5be049cc9e26f4662f8451b2
2024-12-16T00:13:17,742 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/namespace/cb2fa9e4995d7f54e1d8bed7be70c3c1/info/69796aab5be049cc9e26f4662f8451b2, entries=2, sequenceid=6, filesize=4.9 K
2024-12-16T00:13:17,744 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for cb2fa9e4995d7f54e1d8bed7be70c3c1 in 41ms, sequenceid=6, compaction requested=false
2024-12-16T00:13:17,754 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/namespace/cb2fa9e4995d7f54e1d8bed7be70c3c1/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-16T00:13:17,756 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:13:17,756 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for cb2fa9e4995d7f54e1d8bed7be70c3c1:

2024-12-16T00:13:17,756 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1734307889072.cb2fa9e4995d7f54e1d8bed7be70c3c1.
2024-12-16T00:13:17,757 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 64e3193e86d9d4ff92f22ce4aacde553, disabling compactions & flushes
2024-12-16T00:13:17,757 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:13:17,757 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:13:17,757 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553. after waiting 0 ms
2024-12-16T00:13:17,757 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:13:17,757 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 64e3193e86d9d4ff92f22ce4aacde553 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-16T00:13:17,761 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/.tmp/table/b749d1b48c4b4cecacd5273e95ad49e4 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1734307890364/Put/seqid=0
2024-12-16T00:13:17,761 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/b108e02ef6654b2cbaad9769904eb9a5 is 1080, key is row0029/info:/1734307991628/Put/seqid=0
2024-12-16T00:13:17,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741855_1031 (size=5452)
2024-12-16T00:13:17,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741855_1031 (size=5452)
2024-12-16T00:13:17,769 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741856_1032 (size=8193)
2024-12-16T00:13:17,769 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=232 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/.tmp/table/b749d1b48c4b4cecacd5273e95ad49e4
2024-12-16T00:13:17,769 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741856_1032 (size=8193)
2024-12-16T00:13:17,770 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/b108e02ef6654b2cbaad9769904eb9a5
2024-12-16T00:13:17,778 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/.tmp/info/b108e02ef6654b2cbaad9769904eb9a5 as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/b108e02ef6654b2cbaad9769904eb9a5
2024-12-16T00:13:17,778 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/.tmp/info/49fcd85b62da41699c063aaacdd306d3 as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/info/49fcd85b62da41699c063aaacdd306d3
2024-12-16T00:13:17,787 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/b108e02ef6654b2cbaad9769904eb9a5, entries=3, sequenceid=48, filesize=8.0 K
2024-12-16T00:13:17,787 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/info/49fcd85b62da41699c063aaacdd306d3, entries=20, sequenceid=14, filesize=8.0 K
2024-12-16T00:13:17,788 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 64e3193e86d9d4ff92f22ce4aacde553 in 31ms, sequenceid=48, compaction requested=true
2024-12-16T00:13:17,788 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/.tmp/table/b749d1b48c4b4cecacd5273e95ad49e4 as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/table/b749d1b48c4b4cecacd5273e95ad49e4
2024-12-16T00:13:17,788 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/63bbc235a5784cad975740fd7dc1d215, hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/55765891fb584f2c8f37fef15c9c645b, hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/a98f4cae3a164d7badbe12fad1e08b45] to archive
2024-12-16T00:13:17,791 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-16T00:13:17,796 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/63bbc235a5784cad975740fd7dc1d215 to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/archive/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/63bbc235a5784cad975740fd7dc1d215
2024-12-16T00:13:17,796 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/55765891fb584f2c8f37fef15c9c645b to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/archive/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/55765891fb584f2c8f37fef15c9c645b
2024-12-16T00:13:17,796 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/table/b749d1b48c4b4cecacd5273e95ad49e4, entries=4, sequenceid=14, filesize=5.3 K
2024-12-16T00:13:17,798 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~2.81 KB/2882, heapSize ~5.04 KB/5160, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=14, compaction requested=false
2024-12-16T00:13:17,799 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/a98f4cae3a164d7badbe12fad1e08b45 to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/archive/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/info/a98f4cae3a164d7badbe12fad1e08b45
2024-12-16T00:13:17,804 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1
2024-12-16T00:13:17,805 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-16T00:13:17,805 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-16T00:13:17,805 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:13:17,806 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-16T00:13:17,815 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/data/default/TestLogRolling-testSlowSyncLogRolling/64e3193e86d9d4ff92f22ce4aacde553/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-12-16T00:13:17,816 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:13:17,817 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 64e3193e86d9d4ff92f22ce4aacde553:

2024-12-16T00:13:17,817 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1734307889946.64e3193e86d9d4ff92f22ce4aacde553.
2024-12-16T00:13:17,904 INFO  [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1250): stopping server b878af803003,45425,1734307887205; all regions closed.
2024-12-16T00:13:17,905 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205
2024-12-16T00:13:17,907 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741834_1010 (size=4330)
2024-12-16T00:13:17,907 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741834_1010 (size=4330)
2024-12-16T00:13:17,910 DEBUG [RS:0;b878af803003:45425 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/oldWALs
2024-12-16T00:13:17,910 INFO  [RS:0;b878af803003:45425 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog b878af803003%2C45425%2C1734307887205.meta:.meta(num 1734307888805)
2024-12-16T00:13:17,910 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/WALs/b878af803003,45425,1734307887205
2024-12-16T00:13:17,912 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741851_1027 (size=13066)
2024-12-16T00:13:17,912 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741851_1027 (size=13066)
2024-12-16T00:13:17,917 DEBUG [RS:0;b878af803003:45425 {}] wal.AbstractFSWAL(1071): Moved 3 WAL file(s) to /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/oldWALs
2024-12-16T00:13:17,917 INFO  [RS:0;b878af803003:45425 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog b878af803003%2C45425%2C1734307887205:(num 1734307977597)
2024-12-16T00:13:17,917 DEBUG [RS:0;b878af803003:45425 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:13:17,917 INFO  [RS:0;b878af803003:45425 {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:13:17,918 INFO  [RS:0;b878af803003:45425 {}] hbase.ChoreService(370): Chore service for: regionserver/b878af803003:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown
2024-12-16T00:13:17,918 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:13:17,918 INFO  [RS:0;b878af803003:45425 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45425
2024-12-16T00:13:17,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:13:17,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b878af803003,45425,1734307887205
2024-12-16T00:13:17,938 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b878af803003,45425,1734307887205]
2024-12-16T00:13:17,939 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing b878af803003,45425,1734307887205; numProcessing=1
2024-12-16T00:13:17,946 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/b878af803003,45425,1734307887205 already deleted, retry=false
2024-12-16T00:13:17,947 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; b878af803003,45425,1734307887205 expired; onlineServers=0
2024-12-16T00:13:17,947 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,44821,1734307886455' *****
2024-12-16T00:13:17,947 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-16T00:13:17,947 DEBUG [M:0;b878af803003:44821 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74984c82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:13:17,947 INFO  [M:0;b878af803003:44821 {}] regionserver.HRegionServer(1224): stopping server b878af803003,44821,1734307886455
2024-12-16T00:13:17,947 INFO  [M:0;b878af803003:44821 {}] regionserver.HRegionServer(1250): stopping server b878af803003,44821,1734307886455; all regions closed.
2024-12-16T00:13:17,947 DEBUG [M:0;b878af803003:44821 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:13:17,947 DEBUG [M:0;b878af803003:44821 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-16T00:13:17,947 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-16T00:13:17,947 DEBUG [M:0;b878af803003:44821 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-16T00:13:17,947 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734307888089 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734307888089,5,FailOnTimeoutGroup]
2024-12-16T00:13:17,947 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734307888089 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734307888089,5,FailOnTimeoutGroup]
2024-12-16T00:13:17,948 INFO  [M:0;b878af803003:44821 {}] hbase.ChoreService(370): Chore service for: master/b878af803003:0 had [] on shutdown
2024-12-16T00:13:17,948 DEBUG [M:0;b878af803003:44821 {}] master.HMaster(1733): Stopping service threads
2024-12-16T00:13:17,948 INFO  [M:0;b878af803003:44821 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-16T00:13:17,948 INFO  [M:0;b878af803003:44821 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-16T00:13:17,949 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-16T00:13:17,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-16T00:13:17,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:17,955 DEBUG [M:0;b878af803003:44821 {}] zookeeper.ZKUtil(347): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-16T00:13:17,955 WARN  [M:0;b878af803003:44821 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-16T00:13:17,955 INFO  [M:0;b878af803003:44821 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-16T00:13:17,955 INFO  [M:0;b878af803003:44821 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-16T00:13:17,955 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:13:17,955 DEBUG [M:0;b878af803003:44821 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:13:17,955 INFO  [M:0;b878af803003:44821 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:13:17,955 DEBUG [M:0;b878af803003:44821 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:13:17,955 DEBUG [M:0;b878af803003:44821 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:13:17,956 DEBUG [M:0;b878af803003:44821 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:13:17,956 INFO  [M:0;b878af803003:44821 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.21 KB heapSize=50.14 KB
2024-12-16T00:13:17,971 DEBUG [M:0;b878af803003:44821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/880bfe7447774699924e88391fcfc2ac is 82, key is hbase:meta,,1/info:regioninfo/1734307888910/Put/seqid=0
2024-12-16T00:13:17,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741857_1033 (size=5672)
2024-12-16T00:13:17,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741857_1033 (size=5672)
2024-12-16T00:13:17,977 INFO  [M:0;b878af803003:44821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/880bfe7447774699924e88391fcfc2ac
2024-12-16T00:13:17,997 DEBUG [M:0;b878af803003:44821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9a3240df52e04d05a077ad7704aac7bd is 766, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1734307890428/Put/seqid=0
2024-12-16T00:13:18,002 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741858_1034 (size=6426)
2024-12-16T00:13:18,002 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741858_1034 (size=6426)
2024-12-16T00:13:18,003 INFO  [M:0;b878af803003:44821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.61 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9a3240df52e04d05a077ad7704aac7bd
2024-12-16T00:13:18,009 INFO  [M:0;b878af803003:44821 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9a3240df52e04d05a077ad7704aac7bd
2024-12-16T00:13:18,024 DEBUG [M:0;b878af803003:44821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a168eb5443954548bc32ada537fb2514 is 69, key is b878af803003,45425,1734307887205/rs:state/1734307888175/Put/seqid=0
2024-12-16T00:13:18,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741859_1035 (size=5156)
2024-12-16T00:13:18,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741859_1035 (size=5156)
2024-12-16T00:13:18,030 INFO  [M:0;b878af803003:44821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a168eb5443954548bc32ada537fb2514
2024-12-16T00:13:18,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:13:18,038 INFO  [RS:0;b878af803003:45425 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,45425,1734307887205; zookeeper connection closed.
2024-12-16T00:13:18,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45425-0x1002c2d4f110001, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:13:18,039 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1317a4e8 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1317a4e8
2024-12-16T00:13:18,039 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-16T00:13:18,050 DEBUG [M:0;b878af803003:44821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d4d07e169739454b882d9803e6ac39c5 is 52, key is load_balancer_on/state:d/1734307889931/Put/seqid=0
2024-12-16T00:13:18,055 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741860_1036 (size=5056)
2024-12-16T00:13:18,056 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741860_1036 (size=5056)
2024-12-16T00:13:18,056 INFO  [M:0;b878af803003:44821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d4d07e169739454b882d9803e6ac39c5
2024-12-16T00:13:18,063 DEBUG [M:0;b878af803003:44821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/880bfe7447774699924e88391fcfc2ac as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/880bfe7447774699924e88391fcfc2ac
2024-12-16T00:13:18,070 INFO  [M:0;b878af803003:44821 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/880bfe7447774699924e88391fcfc2ac, entries=8, sequenceid=104, filesize=5.5 K
2024-12-16T00:13:18,072 DEBUG [M:0;b878af803003:44821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9a3240df52e04d05a077ad7704aac7bd as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9a3240df52e04d05a077ad7704aac7bd
2024-12-16T00:13:18,078 INFO  [M:0;b878af803003:44821 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9a3240df52e04d05a077ad7704aac7bd
2024-12-16T00:13:18,078 INFO  [M:0;b878af803003:44821 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9a3240df52e04d05a077ad7704aac7bd, entries=11, sequenceid=104, filesize=6.3 K
2024-12-16T00:13:18,079 DEBUG [M:0;b878af803003:44821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a168eb5443954548bc32ada537fb2514 as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a168eb5443954548bc32ada537fb2514
2024-12-16T00:13:18,086 INFO  [M:0;b878af803003:44821 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a168eb5443954548bc32ada537fb2514, entries=1, sequenceid=104, filesize=5.0 K
2024-12-16T00:13:18,087 DEBUG [M:0;b878af803003:44821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d4d07e169739454b882d9803e6ac39c5 as hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d4d07e169739454b882d9803e6ac39c5
2024-12-16T00:13:18,093 INFO  [M:0;b878af803003:44821 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d4d07e169739454b882d9803e6ac39c5, entries=1, sequenceid=104, filesize=4.9 K
2024-12-16T00:13:18,094 INFO  [M:0;b878af803003:44821 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.21 KB/41173, heapSize ~50.08 KB/51280, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 138ms, sequenceid=104, compaction requested=false
2024-12-16T00:13:18,096 INFO  [M:0;b878af803003:44821 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:13:18,096 DEBUG [M:0;b878af803003:44821 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:13:18,096 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/MasterData/WALs/b878af803003,44821,1734307886455
2024-12-16T00:13:18,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40809 is added to blk_1073741830_1006 (size=48474)
2024-12-16T00:13:18,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34417 is added to blk_1073741830_1006 (size=48474)
2024-12-16T00:13:18,099 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:13:18,099 INFO  [M:0;b878af803003:44821 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-16T00:13:18,099 INFO  [M:0;b878af803003:44821 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:44821
2024-12-16T00:13:18,105 DEBUG [M:0;b878af803003:44821 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/b878af803003,44821,1734307886455 already deleted, retry=false
2024-12-16T00:13:18,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:13:18,214 INFO  [M:0;b878af803003:44821 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,44821,1734307886455; zookeeper connection closed.
2024-12-16T00:13:18,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44821-0x1002c2d4f110000, quorum=127.0.0.1:58822, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:13:18,251 INFO  [regionserver/b878af803003:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:13:18,256 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4e5af57{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:18,259 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@68aaa55{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:13:18,260 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:13:18,260 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f9d2cc3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:13:18,260 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22a0043c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/hadoop.log.dir/,STOPPED}
2024-12-16T00:13:18,263 WARN  [BP-1593260936-172.17.0.2-1734307883021 heartbeating to localhost/127.0.0.1:33445 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:13:18,263 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:13:18,263 WARN  [BP-1593260936-172.17.0.2-1734307883021 heartbeating to localhost/127.0.0.1:33445 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1593260936-172.17.0.2-1734307883021 (Datanode Uuid 109aa2fb-53fa-453b-9319-bb9728ab870a) service to localhost/127.0.0.1:33445
2024-12-16T00:13:18,263 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:13:18,265 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54/dfs/data/data3/current/BP-1593260936-172.17.0.2-1734307883021 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:18,265 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54/dfs/data/data4/current/BP-1593260936-172.17.0.2-1734307883021 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:18,266 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:13:18,268 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64b46cf2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:18,269 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1872bb43{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:13:18,269 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:13:18,269 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@44404b82{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:13:18,269 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e40bccb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/hadoop.log.dir/,STOPPED}
2024-12-16T00:13:18,271 WARN  [BP-1593260936-172.17.0.2-1734307883021 heartbeating to localhost/127.0.0.1:33445 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:13:18,271 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:13:18,271 WARN  [BP-1593260936-172.17.0.2-1734307883021 heartbeating to localhost/127.0.0.1:33445 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1593260936-172.17.0.2-1734307883021 (Datanode Uuid 05f02631-6a58-4943-a09b-77009a366199) service to localhost/127.0.0.1:33445
2024-12-16T00:13:18,271 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:13:18,272 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54/dfs/data/data1/current/BP-1593260936-172.17.0.2-1734307883021 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:18,272 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/cluster_9b88641f-6196-d65c-cc74-cec93bd62e54/dfs/data/data2/current/BP-1593260936-172.17.0.2-1734307883021 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:18,273 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:13:18,283 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@805400f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:13:18,284 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6567b82c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:13:18,284 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:13:18,284 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fabcd45{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:13:18,284 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17bd5227{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/hadoop.log.dir/,STOPPED}
2024-12-16T00:13:18,293 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-16T00:13:18,325 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-16T00:13:18,331 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=64 (was 12)
Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (679588824) connection to localhost/127.0.0.1:33445 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/b878af803003:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33445
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (679588824) connection to localhost/127.0.0.1:33445 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: master/b878af803003:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@6b7dcb67
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:33445
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (679588824) connection to localhost/127.0.0.1:33445 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/b878af803003:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33445
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33445
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:33445
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=408 (was 286) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=49 (was 142), ProcessCount=11 (was 11), AvailableMemoryMB=12492 (was 13068)
2024-12-16T00:13:18,336 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=65, OpenFileDescriptor=408, MaxFileDescriptor=1048576, SystemLoadAverage=49, ProcessCount=11, AvailableMemoryMB=12491
2024-12-16T00:13:18,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-16T00:13:18,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/hadoop.log.dir so I do NOT create it in target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9
2024-12-16T00:13:18,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/7fcbd352-bce8-f8c3-096f-eebbe8ab55ef/hadoop.tmp.dir so I do NOT create it in target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9
2024-12-16T00:13:18,337 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5, deleteOnExit=true
2024-12-16T00:13:18,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-16T00:13:18,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/test.cache.data in system properties and HBase conf
2024-12-16T00:13:18,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.tmp.dir in system properties and HBase conf
2024-12-16T00:13:18,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir in system properties and HBase conf
2024-12-16T00:13:18,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-16T00:13:18,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-16T00:13:18,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-16T00:13:18,338 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-16T00:13:18,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:13:18,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:13:18,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-16T00:13:18,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:13:18,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-16T00:13:18,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-16T00:13:18,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:13:18,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:13:18,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-16T00:13:18,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/nfs.dump.dir in system properties and HBase conf
2024-12-16T00:13:18,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/java.io.tmpdir in system properties and HBase conf
2024-12-16T00:13:18,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:13:18,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-16T00:13:18,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-16T00:13:18,352 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:13:18,626 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:13:18,631 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:13:18,632 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:13:18,632 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:13:18,633 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-16T00:13:18,633 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:13:18,634 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c4a27fe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:13:18,634 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25d29461{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:13:18,724 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6f234dd7{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/java.io.tmpdir/jetty-localhost-45105-hadoop-hdfs-3_4_1-tests_jar-_-any-3874269150618655183/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:13:18,725 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3afa3b9{HTTP/1.1, (http/1.1)}{localhost:45105}
2024-12-16T00:13:18,725 INFO  [Time-limited test {}] server.Server(415): Started @117390ms
2024-12-16T00:13:18,736 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:13:18,968 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:13:18,971 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:13:18,972 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:13:18,972 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:13:18,972 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-16T00:13:18,972 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b2f8217{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:13:18,973 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f590214{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:13:19,062 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3ca4a1d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/java.io.tmpdir/jetty-localhost-43773-hadoop-hdfs-3_4_1-tests_jar-_-any-4346546570875503458/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:19,063 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4dfe23a3{HTTP/1.1, (http/1.1)}{localhost:43773}
2024-12-16T00:13:19,063 INFO  [Time-limited test {}] server.Server(415): Started @117728ms
2024-12-16T00:13:19,064 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:13:19,096 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:13:19,100 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:13:19,101 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:13:19,101 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:13:19,101 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:13:19,102 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ed21f9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:13:19,103 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48cc4a03{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:13:19,200 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7a270e7d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/java.io.tmpdir/jetty-localhost-45761-hadoop-hdfs-3_4_1-tests_jar-_-any-8658000079808741361/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:19,200 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@75ee43d4{HTTP/1.1, (http/1.1)}{localhost:45761}
2024-12-16T00:13:19,200 INFO  [Time-limited test {}] server.Server(415): Started @117865ms
2024-12-16T00:13:19,202 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:13:19,718 WARN  [Thread-467 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data1/current/BP-1525461463-172.17.0.2-1734307998365/current, will proceed with Du for space computation calculation, 
2024-12-16T00:13:19,719 WARN  [Thread-468 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data2/current/BP-1525461463-172.17.0.2-1734307998365/current, will proceed with Du for space computation calculation, 
2024-12-16T00:13:19,740 WARN  [Thread-432 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:13:19,743 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4b3a152ce62892b3 with lease ID 0x125ba5608a7cc509: Processing first storage report for DS-f0103ffb-8953-400c-8557-7ca5936be765 from datanode DatanodeRegistration(127.0.0.1:43815, datanodeUuid=fdbe45cc-ce4f-48e3-bbfe-26744033b547, infoPort=41127, infoSecurePort=0, ipcPort=38427, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365)
2024-12-16T00:13:19,743 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4b3a152ce62892b3 with lease ID 0x125ba5608a7cc509: from storage DS-f0103ffb-8953-400c-8557-7ca5936be765 node DatanodeRegistration(127.0.0.1:43815, datanodeUuid=fdbe45cc-ce4f-48e3-bbfe-26744033b547, infoPort=41127, infoSecurePort=0, ipcPort=38427, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-16T00:13:19,743 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4b3a152ce62892b3 with lease ID 0x125ba5608a7cc509: Processing first storage report for DS-03c1c6de-8669-47c8-86ef-a5b7b76892d5 from datanode DatanodeRegistration(127.0.0.1:43815, datanodeUuid=fdbe45cc-ce4f-48e3-bbfe-26744033b547, infoPort=41127, infoSecurePort=0, ipcPort=38427, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365)
2024-12-16T00:13:19,743 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4b3a152ce62892b3 with lease ID 0x125ba5608a7cc509: from storage DS-03c1c6de-8669-47c8-86ef-a5b7b76892d5 node DatanodeRegistration(127.0.0.1:43815, datanodeUuid=fdbe45cc-ce4f-48e3-bbfe-26744033b547, infoPort=41127, infoSecurePort=0, ipcPort=38427, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:13:19,916 WARN  [Thread-479 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data3/current/BP-1525461463-172.17.0.2-1734307998365/current, will proceed with Du for space computation calculation, 
2024-12-16T00:13:19,917 WARN  [Thread-480 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data4/current/BP-1525461463-172.17.0.2-1734307998365/current, will proceed with Du for space computation calculation, 
2024-12-16T00:13:19,936 WARN  [Thread-455 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:13:19,939 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaeffee75d3ca1ac8 with lease ID 0x125ba5608a7cc50a: Processing first storage report for DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5 from datanode DatanodeRegistration(127.0.0.1:38223, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=36599, infoSecurePort=0, ipcPort=41465, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365)
2024-12-16T00:13:19,939 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaeffee75d3ca1ac8 with lease ID 0x125ba5608a7cc50a: from storage DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5 node DatanodeRegistration(127.0.0.1:38223, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=36599, infoSecurePort=0, ipcPort=41465, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-16T00:13:19,939 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaeffee75d3ca1ac8 with lease ID 0x125ba5608a7cc50a: Processing first storage report for DS-02cd5796-a69e-49e8-9424-2062466f8ed7 from datanode DatanodeRegistration(127.0.0.1:38223, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=36599, infoSecurePort=0, ipcPort=41465, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365)
2024-12-16T00:13:19,939 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaeffee75d3ca1ac8 with lease ID 0x125ba5608a7cc50a: from storage DS-02cd5796-a69e-49e8-9424-2062466f8ed7 node DatanodeRegistration(127.0.0.1:38223, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=36599, infoSecurePort=0, ipcPort=41465, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:13:19,944 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9
2024-12-16T00:13:19,946 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/zookeeper_0, clientPort=50614, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-16T00:13:19,947 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=50614
2024-12-16T00:13:19,947 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:13:19,949 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:13:19,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43815 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:13:19,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38223 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:13:19,961 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db with version=8
2024-12-16T00:13:19,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/hbase-staging
2024-12-16T00:13:19,963 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/b878af803003:0 server-side Connection retries=45
2024-12-16T00:13:19,963 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:13:19,963 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:13:19,963 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:13:19,963 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:13:19,963 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:13:19,963 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:13:19,963 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:13:19,964 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:39029
2024-12-16T00:13:19,964 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:13:19,966 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:13:19,968 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:39029 connecting to ZooKeeper ensemble=127.0.0.1:50614
2024-12-16T00:13:20,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:390290x0, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:13:20,020 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39029-0x1002c2f0d4a0000 connected
2024-12-16T00:13:20,080 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:13:20,081 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:13:20,082 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:13:20,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39029
2024-12-16T00:13:20,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39029
2024-12-16T00:13:20,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39029
2024-12-16T00:13:20,084 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39029
2024-12-16T00:13:20,084 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39029
2024-12-16T00:13:20,084 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db, hbase.cluster.distributed=false
2024-12-16T00:13:20,100 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/b878af803003:0 server-side Connection retries=45
2024-12-16T00:13:20,100 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:13:20,100 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:13:20,100 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:13:20,100 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:13:20,100 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:13:20,100 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:13:20,100 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:13:20,101 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:39441
2024-12-16T00:13:20,101 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-16T00:13:20,103 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-16T00:13:20,104 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:13:20,106 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:13:20,109 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:39441 connecting to ZooKeeper ensemble=127.0.0.1:50614
2024-12-16T00:13:20,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394410x0, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:13:20,122 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:394410x0, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:13:20,122 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39441-0x1002c2f0d4a0001 connected
2024-12-16T00:13:20,123 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:13:20,124 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:13:20,125 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39441
2024-12-16T00:13:20,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39441
2024-12-16T00:13:20,129 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39441
2024-12-16T00:13:20,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39441
2024-12-16T00:13:20,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39441
2024-12-16T00:13:20,132 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/b878af803003,39029,1734307999962
2024-12-16T00:13:20,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:13:20,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:13:20,139 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b878af803003,39029,1734307999962
2024-12-16T00:13:20,144 DEBUG [M:0;b878af803003:39029 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b878af803003:39029
2024-12-16T00:13:20,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:13:20,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:13:20,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,147 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:13:20,147 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b878af803003,39029,1734307999962 from backup master directory
2024-12-16T00:13:20,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b878af803003,39029,1734307999962
2024-12-16T00:13:20,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:13:20,155 WARN  [master/b878af803003:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:13:20,155 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:13:20,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:13:20,155 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b878af803003,39029,1734307999962
2024-12-16T00:13:20,167 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43815 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:13:20,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38223 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:13:20,170 DEBUG [master/b878af803003:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/hbase.id with ID: 3e93138e-cc79-47f0-9fb4-2f1321d85178
2024-12-16T00:13:20,185 INFO  [master/b878af803003:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:13:20,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38223 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:13:20,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43815 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:13:20,205 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-16T00:13:20,206 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-16T00:13:20,206 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:13:20,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43815 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:13:20,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38223 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:13:20,217 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store
2024-12-16T00:13:20,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43815 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:13:20,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38223 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:13:20,225 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:13:20,226 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:13:20,226 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:13:20,226 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:13:20,226 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:13:20,226 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:13:20,226 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:13:20,226 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:13:20,227 WARN  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/.initializing
2024-12-16T00:13:20,227 DEBUG [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962
2024-12-16T00:13:20,230 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C39029%2C1734307999962, suffix=, logDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962, archiveDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/oldWALs, maxLogs=10
2024-12-16T00:13:20,231 INFO  [master/b878af803003:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C39029%2C1734307999962.1734308000230
2024-12-16T00:13:20,240 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308000230
2024-12-16T00:13:20,240 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41127:41127),(127.0.0.1/127.0.0.1:36599:36599)]
2024-12-16T00:13:20,240 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:13:20,240 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:13:20,240 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:13:20,240 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:13:20,243 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:13:20,245 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-16T00:13:20,245 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:20,246 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:13:20,246 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:13:20,247 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-16T00:13:20,248 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:20,248 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:13:20,249 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:13:20,250 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-16T00:13:20,250 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:20,251 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:13:20,251 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:13:20,253 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-16T00:13:20,253 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:20,254 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:13:20,256 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:13:20,256 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:13:20,258 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-16T00:13:20,259 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:13:20,263 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:13:20,263 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=706610, jitterRate=-0.10149902105331421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-16T00:13:20,264 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:13:20,265 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-16T00:13:20,269 DEBUG [master/b878af803003:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d2fd6cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:13:20,270 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-16T00:13:20,271 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-16T00:13:20,271 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-16T00:13:20,271 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-16T00:13:20,272 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-16T00:13:20,272 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-16T00:13:20,272 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-16T00:13:20,275 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-16T00:13:20,275 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-16T00:13:20,280 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-16T00:13:20,280 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-16T00:13:20,281 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-16T00:13:20,288 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-16T00:13:20,288 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-16T00:13:20,289 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-16T00:13:20,296 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-16T00:13:20,297 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-16T00:13:20,305 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-16T00:13:20,308 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-16T00:13:20,313 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-16T00:13:20,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:13:20,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:13:20,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,323 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=b878af803003,39029,1734307999962, sessionid=0x1002c2f0d4a0000, setting cluster-up flag (Was=false)
2024-12-16T00:13:20,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,363 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-16T00:13:20,365 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,39029,1734307999962
2024-12-16T00:13:20,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,405 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-16T00:13:20,407 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,39029,1734307999962
2024-12-16T00:13:20,411 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-16T00:13:20,411 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-16T00:13:20,411 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-16T00:13:20,411 DEBUG [master/b878af803003:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b878af803003,39029,1734307999962
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-16T00:13:20,412 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:13:20,412 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:13:20,412 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:13:20,412 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:13:20,412 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b878af803003:0, corePoolSize=10, maxPoolSize=10
2024-12-16T00:13:20,412 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,412 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:13:20,412 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,413 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734308030413
2024-12-16T00:13:20,413 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-16T00:13:20,413 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-16T00:13:20,413 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-16T00:13:20,413 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-16T00:13:20,413 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-16T00:13:20,413 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-16T00:13:20,414 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:20,414 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:13:20,414 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-16T00:13:20,414 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-16T00:13:20,414 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-16T00:13:20,414 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-16T00:13:20,414 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-16T00:13:20,414 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-16T00:13:20,415 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308000414,5,FailOnTimeoutGroup]
2024-12-16T00:13:20,415 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308000415,5,FailOnTimeoutGroup]
2024-12-16T00:13:20,415 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:20,415 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-16T00:13:20,415 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:20,415 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:20,415 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:20,415 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:13:20,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43815 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:13:20,425 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38223 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:13:20,426 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-16T00:13:20,426 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db
2024-12-16T00:13:20,433 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38223 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:13:20,433 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43815 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:13:20,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:13:20,435 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:13:20,437 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:13:20,437 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:20,438 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:13:20,438 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:13:20,440 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:13:20,440 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:20,440 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:13:20,440 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:13:20,442 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:13:20,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:20,443 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:13:20,444 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/hbase/meta/1588230740
2024-12-16T00:13:20,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/hbase/meta/1588230740
2024-12-16T00:13:20,447 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:13:20,448 DEBUG [RS:0;b878af803003:39441 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b878af803003:39441
2024-12-16T00:13:20,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:13:20,449 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1008): ClusterId : 3e93138e-cc79-47f0-9fb4-2f1321d85178
2024-12-16T00:13:20,449 DEBUG [RS:0;b878af803003:39441 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-16T00:13:20,451 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:13:20,452 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=873675, jitterRate=0.11093541979789734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:13:20,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:13:20,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:13:20,453 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:13:20,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:13:20,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:13:20,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:13:20,453 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-16T00:13:20,454 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:13:20,455 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:13:20,455 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-16T00:13:20,455 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-16T00:13:20,455 DEBUG [RS:0;b878af803003:39441 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-16T00:13:20,455 DEBUG [RS:0;b878af803003:39441 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-16T00:13:20,457 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-16T00:13:20,458 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-16T00:13:20,464 DEBUG [RS:0;b878af803003:39441 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-16T00:13:20,464 DEBUG [RS:0;b878af803003:39441 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f96c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:13:20,464 DEBUG [RS:0;b878af803003:39441 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@329a91dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:13:20,465 INFO  [RS:0;b878af803003:39441 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-16T00:13:20,465 INFO  [RS:0;b878af803003:39441 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-16T00:13:20,465 DEBUG [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-16T00:13:20,465 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(3073): reportForDuty to master=b878af803003,39029,1734307999962 with isa=b878af803003/172.17.0.2:39441, startcode=1734308000099
2024-12-16T00:13:20,465 DEBUG [RS:0;b878af803003:39441 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:13:20,468 INFO  [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47309, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:13:20,468 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39029 {}] master.ServerManager(332): Checking decommissioned status of RegionServer b878af803003,39441,1734308000099
2024-12-16T00:13:20,468 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39029 {}] master.ServerManager(486): Registering regionserver=b878af803003,39441,1734308000099
2024-12-16T00:13:20,470 DEBUG [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db
2024-12-16T00:13:20,470 DEBUG [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:37443
2024-12-16T00:13:20,470 DEBUG [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-16T00:13:20,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:13:20,480 DEBUG [RS:0;b878af803003:39441 {}] zookeeper.ZKUtil(111): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b878af803003,39441,1734308000099
2024-12-16T00:13:20,480 WARN  [RS:0;b878af803003:39441 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:13:20,480 INFO  [RS:0;b878af803003:39441 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:13:20,481 DEBUG [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099
2024-12-16T00:13:20,481 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b878af803003,39441,1734308000099]
2024-12-16T00:13:20,484 DEBUG [RS:0;b878af803003:39441 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-16T00:13:20,485 INFO  [RS:0;b878af803003:39441 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-16T00:13:20,487 INFO  [RS:0;b878af803003:39441 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-16T00:13:20,487 INFO  [RS:0;b878af803003:39441 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-16T00:13:20,488 INFO  [RS:0;b878af803003:39441 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:20,488 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-16T00:13:20,490 INFO  [RS:0;b878af803003:39441 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:20,490 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,490 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,490 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,490 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,491 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,491 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:13:20,491 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,491 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,491 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,491 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,491 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:20,491 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:13:20,492 DEBUG [RS:0;b878af803003:39441 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:13:20,492 INFO  [RS:0;b878af803003:39441 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:20,492 INFO  [RS:0;b878af803003:39441 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:20,492 INFO  [RS:0;b878af803003:39441 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:20,492 INFO  [RS:0;b878af803003:39441 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:20,492 INFO  [RS:0;b878af803003:39441 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,39441,1734308000099-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:13:20,505 INFO  [RS:0;b878af803003:39441 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-16T00:13:20,506 INFO  [RS:0;b878af803003:39441 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,39441,1734308000099-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:20,517 INFO  [RS:0;b878af803003:39441 {}] regionserver.Replication(204): b878af803003,39441,1734308000099 started
2024-12-16T00:13:20,518 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1767): Serving as b878af803003,39441,1734308000099, RpcServer on b878af803003/172.17.0.2:39441, sessionid=0x1002c2f0d4a0001
2024-12-16T00:13:20,518 DEBUG [RS:0;b878af803003:39441 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-16T00:13:20,518 DEBUG [RS:0;b878af803003:39441 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b878af803003,39441,1734308000099
2024-12-16T00:13:20,518 DEBUG [RS:0;b878af803003:39441 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,39441,1734308000099'
2024-12-16T00:13:20,518 DEBUG [RS:0;b878af803003:39441 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-16T00:13:20,518 DEBUG [RS:0;b878af803003:39441 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-16T00:13:20,519 DEBUG [RS:0;b878af803003:39441 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-16T00:13:20,519 DEBUG [RS:0;b878af803003:39441 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-16T00:13:20,519 DEBUG [RS:0;b878af803003:39441 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b878af803003,39441,1734308000099
2024-12-16T00:13:20,519 DEBUG [RS:0;b878af803003:39441 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,39441,1734308000099'
2024-12-16T00:13:20,519 DEBUG [RS:0;b878af803003:39441 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-16T00:13:20,519 DEBUG [RS:0;b878af803003:39441 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-16T00:13:20,520 DEBUG [RS:0;b878af803003:39441 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-16T00:13:20,520 INFO  [RS:0;b878af803003:39441 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-16T00:13:20,520 INFO  [RS:0;b878af803003:39441 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-16T00:13:20,608 WARN  [b878af803003:39029 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-16T00:13:20,624 INFO  [RS:0;b878af803003:39441 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C39441%2C1734308000099, suffix=, logDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099, archiveDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/oldWALs, maxLogs=32
2024-12-16T00:13:20,629 INFO  [RS:0;b878af803003:39441 {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C39441%2C1734308000099.1734308000629
2024-12-16T00:13:20,640 INFO  [RS:0;b878af803003:39441 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629
2024-12-16T00:13:20,640 DEBUG [RS:0;b878af803003:39441 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36599:36599),(127.0.0.1/127.0.0.1:41127:41127)]
2024-12-16T00:13:20,858 DEBUG [b878af803003:39029 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-16T00:13:20,859 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b878af803003,39441,1734308000099
2024-12-16T00:13:20,863 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,39441,1734308000099, state=OPENING
2024-12-16T00:13:20,896 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-16T00:13:20,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:20,905 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=b878af803003,39441,1734308000099}]
2024-12-16T00:13:20,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:13:20,906 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:13:21,060 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to b878af803003,39441,1734308000099
2024-12-16T00:13:21,060 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-16T00:13:21,066 INFO  [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43600, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-16T00:13:21,073 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-16T00:13:21,073 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:13:21,076 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C39441%2C1734308000099.meta, suffix=.meta, logDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099, archiveDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/oldWALs, maxLogs=32
2024-12-16T00:13:21,078 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
2024-12-16T00:13:21,085 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
2024-12-16T00:13:21,085 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36599:36599),(127.0.0.1/127.0.0.1:41127:41127)]
2024-12-16T00:13:21,085 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:13:21,085 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-16T00:13:21,086 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-16T00:13:21,086 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-16T00:13:21,086 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-16T00:13:21,086 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:13:21,086 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-16T00:13:21,086 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-16T00:13:21,088 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:13:21,089 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:13:21,089 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:21,090 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:13:21,090 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:13:21,091 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:13:21,091 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:21,092 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:13:21,092 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:13:21,093 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:13:21,093 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:21,093 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:13:21,094 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/hbase/meta/1588230740
2024-12-16T00:13:21,096 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/hbase/meta/1588230740
2024-12-16T00:13:21,098 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:13:21,100 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:13:21,100 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=736890, jitterRate=-0.06299619376659393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:13:21,101 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:13:21,102 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734308001060
2024-12-16T00:13:21,104 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-16T00:13:21,104 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-16T00:13:21,105 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,39441,1734308000099
2024-12-16T00:13:21,105 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,39441,1734308000099, state=OPEN
2024-12-16T00:13:21,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:13:21,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:13:21,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:13:21,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:13:21,141 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-16T00:13:21,141 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=b878af803003,39441,1734308000099 in 231 msec
2024-12-16T00:13:21,145 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-16T00:13:21,146 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 686 msec
2024-12-16T00:13:21,150 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 738 msec
2024-12-16T00:13:21,150 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734308001150, completionTime=-1
2024-12-16T00:13:21,150 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-16T00:13:21,150 DEBUG [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-16T00:13:21,151 DEBUG [hconnection-0x21da2529-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:13:21,152 INFO  [RS-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43602, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:13:21,153 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-16T00:13:21,153 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734308061153
2024-12-16T00:13:21,153 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734308121153
2024-12-16T00:13:21,153 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec
2024-12-16T00:13:21,180 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,39029,1734307999962-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,180 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,39029,1734307999962-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,180 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,39029,1734307999962-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,180 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b878af803003:39029, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,181 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,181 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-16T00:13:21,181 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:13:21,183 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-16T00:13:21,183 DEBUG [master/b878af803003:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-16T00:13:21,184 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-16T00:13:21,185 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:21,186 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-16T00:13:21,196 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38223 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:13:21,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43815 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:13:21,199 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 80495c60a5e712e96c604963e7816d5b, NAME => 'hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db
2024-12-16T00:13:21,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38223 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:13:21,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43815 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:13:21,206 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:13:21,206 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 80495c60a5e712e96c604963e7816d5b, disabling compactions & flushes
2024-12-16T00:13:21,206 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:13:21,206 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:13:21,206 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b. after waiting 0 ms
2024-12-16T00:13:21,206 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:13:21,206 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:13:21,206 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 80495c60a5e712e96c604963e7816d5b:

2024-12-16T00:13:21,208 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-16T00:13:21,208 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734308001208"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734308001208"}]},"ts":"1734308001208"}
2024-12-16T00:13:21,210 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-16T00:13:21,211 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-16T00:13:21,211 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308001211"}]},"ts":"1734308001211"}
2024-12-16T00:13:21,213 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-16T00:13:21,230 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=80495c60a5e712e96c604963e7816d5b, ASSIGN}]
2024-12-16T00:13:21,232 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=80495c60a5e712e96c604963e7816d5b, ASSIGN
2024-12-16T00:13:21,233 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=80495c60a5e712e96c604963e7816d5b, ASSIGN; state=OFFLINE, location=b878af803003,39441,1734308000099; forceNewPlan=false, retain=false
2024-12-16T00:13:21,384 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=80495c60a5e712e96c604963e7816d5b, regionState=OPENING, regionLocation=b878af803003,39441,1734308000099
2024-12-16T00:13:21,391 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 80495c60a5e712e96c604963e7816d5b, server=b878af803003,39441,1734308000099}]
2024-12-16T00:13:21,546 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to b878af803003,39441,1734308000099
2024-12-16T00:13:21,554 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:13:21,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 80495c60a5e712e96c604963e7816d5b, NAME => 'hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:13:21,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 80495c60a5e712e96c604963e7816d5b
2024-12-16T00:13:21,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:13:21,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 80495c60a5e712e96c604963e7816d5b
2024-12-16T00:13:21,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 80495c60a5e712e96c604963e7816d5b
2024-12-16T00:13:21,559 INFO  [StoreOpener-80495c60a5e712e96c604963e7816d5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 80495c60a5e712e96c604963e7816d5b 
2024-12-16T00:13:21,561 INFO  [StoreOpener-80495c60a5e712e96c604963e7816d5b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80495c60a5e712e96c604963e7816d5b columnFamilyName info
2024-12-16T00:13:21,561 DEBUG [StoreOpener-80495c60a5e712e96c604963e7816d5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:21,561 INFO  [StoreOpener-80495c60a5e712e96c604963e7816d5b-1 {}] regionserver.HStore(327): Store=80495c60a5e712e96c604963e7816d5b/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:13:21,562 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/hbase/namespace/80495c60a5e712e96c604963e7816d5b
2024-12-16T00:13:21,562 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/hbase/namespace/80495c60a5e712e96c604963e7816d5b
2024-12-16T00:13:21,564 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 80495c60a5e712e96c604963e7816d5b
2024-12-16T00:13:21,566 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/hbase/namespace/80495c60a5e712e96c604963e7816d5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:13:21,567 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 80495c60a5e712e96c604963e7816d5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=844135, jitterRate=0.07337430119514465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:13:21,568 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 80495c60a5e712e96c604963e7816d5b:

2024-12-16T00:13:21,569 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b., pid=6, masterSystemTime=1734308001546
2024-12-16T00:13:21,571 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:13:21,571 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:13:21,571 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=80495c60a5e712e96c604963e7816d5b, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,39441,1734308000099
2024-12-16T00:13:21,575 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-16T00:13:21,575 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 80495c60a5e712e96c604963e7816d5b, server=b878af803003,39441,1734308000099 in 182 msec
2024-12-16T00:13:21,577 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-16T00:13:21,578 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=80495c60a5e712e96c604963e7816d5b, ASSIGN in 345 msec
2024-12-16T00:13:21,578 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-16T00:13:21,579 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308001578"}]},"ts":"1734308001578"}
2024-12-16T00:13:21,580 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-16T00:13:21,623 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-16T00:13:21,624 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-16T00:13:21,628 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 443 msec
2024-12-16T00:13:21,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:13:21,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:21,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:13:21,636 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-16T00:13:21,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:13:21,666 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 29 msec
2024-12-16T00:13:21,669 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-16T00:13:21,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:13:21,691 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 21 msec
2024-12-16T00:13:21,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-16T00:13:21,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-16T00:13:21,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.575sec
2024-12-16T00:13:21,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-16T00:13:21,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-16T00:13:21,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-16T00:13:21,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-16T00:13:21,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-16T00:13:21,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,39029,1734307999962-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:13:21,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,39029,1734307999962-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-16T00:13:21,732 DEBUG [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-16T00:13:21,732 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-16T00:13:21,732 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,39029,1734307999962-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,737 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3cfd3e0f to 127.0.0.1:50614 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7bafdd00
2024-12-16T00:13:21,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d1b8ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:13:21,749 DEBUG [hconnection-0x125ff5a0-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:13:21,751 INFO  [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43612, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:13:21,753 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=b878af803003,39029,1734307999962
2024-12-16T00:13:21,753 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:13:21,756 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-16T00:13:21,769 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/b878af803003:0 server-side Connection retries=45
2024-12-16T00:13:21,769 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:13:21,769 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:13:21,769 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:13:21,769 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:13:21,769 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:13:21,769 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:13:21,769 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:13:21,770 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36405
2024-12-16T00:13:21,770 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-16T00:13:21,771 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-16T00:13:21,772 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:13:21,774 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:13:21,776 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:36405 connecting to ZooKeeper ensemble=127.0.0.1:50614
2024-12-16T00:13:21,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:364050x0, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:13:21,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:364050x0, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:13:21,780 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36405-0x1002c2f0d4a0003 connected
2024-12-16T00:13:21,781 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36405-0x1002c2f0d4a0003, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-16T00:13:21,782 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36405-0x1002c2f0d4a0003, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:13:21,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36405
2024-12-16T00:13:21,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36405
2024-12-16T00:13:21,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36405
2024-12-16T00:13:21,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36405
2024-12-16T00:13:21,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36405
2024-12-16T00:13:21,786 DEBUG [pool-282-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-16T00:13:21,796 DEBUG [RS:1;b878af803003:36405 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b878af803003:36405
2024-12-16T00:13:21,798 INFO  [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(1008): ClusterId : 3e93138e-cc79-47f0-9fb4-2f1321d85178
2024-12-16T00:13:21,798 DEBUG [RS:1;b878af803003:36405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-16T00:13:21,806 DEBUG [RS:1;b878af803003:36405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-16T00:13:21,806 DEBUG [RS:1;b878af803003:36405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-16T00:13:21,814 DEBUG [RS:1;b878af803003:36405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-16T00:13:21,814 DEBUG [RS:1;b878af803003:36405 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64646c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:13:21,815 DEBUG [RS:1;b878af803003:36405 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c9a6d73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:13:21,815 INFO  [RS:1;b878af803003:36405 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-16T00:13:21,815 INFO  [RS:1;b878af803003:36405 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-16T00:13:21,815 DEBUG [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-16T00:13:21,815 INFO  [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(3073): reportForDuty to master=b878af803003,39029,1734307999962 with isa=b878af803003/172.17.0.2:36405, startcode=1734308001768
2024-12-16T00:13:21,815 DEBUG [RS:1;b878af803003:36405 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:13:21,817 INFO  [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47943, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:13:21,817 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39029 {}] master.ServerManager(332): Checking decommissioned status of RegionServer b878af803003,36405,1734308001768
2024-12-16T00:13:21,817 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39029 {}] master.ServerManager(486): Registering regionserver=b878af803003,36405,1734308001768
2024-12-16T00:13:21,819 DEBUG [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db
2024-12-16T00:13:21,819 DEBUG [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:37443
2024-12-16T00:13:21,819 DEBUG [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-16T00:13:21,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:13:21,875 DEBUG [RS:1;b878af803003:36405 {}] zookeeper.ZKUtil(111): regionserver:36405-0x1002c2f0d4a0003, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b878af803003,36405,1734308001768
2024-12-16T00:13:21,875 WARN  [RS:1;b878af803003:36405 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:13:21,875 INFO  [RS:1;b878af803003:36405 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:13:21,875 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b878af803003,36405,1734308001768]
2024-12-16T00:13:21,875 DEBUG [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,36405,1734308001768
2024-12-16T00:13:21,881 DEBUG [RS:1;b878af803003:36405 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-16T00:13:21,882 INFO  [RS:1;b878af803003:36405 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-16T00:13:21,887 INFO  [RS:1;b878af803003:36405 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-16T00:13:21,888 INFO  [RS:1;b878af803003:36405 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-16T00:13:21,888 INFO  [RS:1;b878af803003:36405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,888 INFO  [RS:1;b878af803003:36405 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-16T00:13:21,889 INFO  [RS:1;b878af803003:36405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,889 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:21,890 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:21,890 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:21,890 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:21,890 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:21,890 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:13:21,890 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:21,890 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:21,890 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:21,890 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:21,890 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:13:21,891 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:13:21,891 DEBUG [RS:1;b878af803003:36405 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:13:21,891 INFO  [RS:1;b878af803003:36405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,891 INFO  [RS:1;b878af803003:36405 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,891 INFO  [RS:1;b878af803003:36405 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,891 INFO  [RS:1;b878af803003:36405 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,891 INFO  [RS:1;b878af803003:36405 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,36405,1734308001768-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:13:21,904 INFO  [RS:1;b878af803003:36405 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-16T00:13:21,904 INFO  [RS:1;b878af803003:36405 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,36405,1734308001768-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:13:21,916 INFO  [RS:1;b878af803003:36405 {}] regionserver.Replication(204): b878af803003,36405,1734308001768 started
2024-12-16T00:13:21,916 INFO  [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(1767): Serving as b878af803003,36405,1734308001768, RpcServer on b878af803003/172.17.0.2:36405, sessionid=0x1002c2f0d4a0003
2024-12-16T00:13:21,916 DEBUG [RS:1;b878af803003:36405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-16T00:13:21,916 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(3355): Started new server=Thread[RS:1;b878af803003:36405,5,FailOnTimeoutGroup]
2024-12-16T00:13:21,916 DEBUG [RS:1;b878af803003:36405 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b878af803003,36405,1734308001768
2024-12-16T00:13:21,916 DEBUG [RS:1;b878af803003:36405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,36405,1734308001768'
2024-12-16T00:13:21,916 DEBUG [RS:1;b878af803003:36405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-16T00:13:21,916 INFO  [Time-limited test {}] wal.TestLogRolling(191): Replication=2
2024-12-16T00:13:21,917 DEBUG [RS:1;b878af803003:36405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-16T00:13:21,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-16T00:13:21,917 DEBUG [RS:1;b878af803003:36405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-16T00:13:21,917 DEBUG [RS:1;b878af803003:36405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-16T00:13:21,918 DEBUG [RS:1;b878af803003:36405 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b878af803003,36405,1734308001768
2024-12-16T00:13:21,918 DEBUG [RS:1;b878af803003:36405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,36405,1734308001768'
2024-12-16T00:13:21,918 DEBUG [RS:1;b878af803003:36405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-16T00:13:21,918 DEBUG [RS:1;b878af803003:36405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-16T00:13:21,918 DEBUG [RS:1;b878af803003:36405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-16T00:13:21,918 INFO  [RS:1;b878af803003:36405 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-16T00:13:21,918 INFO  [RS:1;b878af803003:36405 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-16T00:13:21,919 INFO  [RS-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35248, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-16T00:13:21,920 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39029 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-16T00:13:21,920 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39029 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-16T00:13:21,920 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39029 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-16T00:13:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39029 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-12-16T00:13:21,923 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-12-16T00:13:21,923 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:21,923 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39029 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 9
2024-12-16T00:13:21,924 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-16T00:13:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39029 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-16T00:13:21,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38223 is added to blk_1073741837_1013 (size=393)
2024-12-16T00:13:21,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43815 is added to blk_1073741837_1013 (size=393)
2024-12-16T00:13:21,932 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c7b798bc69dfc2fada33fccf8ca28883, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db
2024-12-16T00:13:21,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43815 is added to blk_1073741838_1014 (size=76)
2024-12-16T00:13:21,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:13:21,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1681): Closing c7b798bc69dfc2fada33fccf8ca28883, disabling compactions & flushes
2024-12-16T00:13:21,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38223 is added to blk_1073741838_1014 (size=76)
2024-12-16T00:13:21,940 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:13:21,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:13:21,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883. after waiting 0 ms
2024-12-16T00:13:21,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:13:21,940 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:13:21,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1635): Region close journal for c7b798bc69dfc2fada33fccf8ca28883:

2024-12-16T00:13:21,941 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-12-16T00:13:21,942 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1734308001941"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734308001941"}]},"ts":"1734308001941"}
2024-12-16T00:13:21,943 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-16T00:13:21,945 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-16T00:13:21,945 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308001945"}]},"ts":"1734308001945"}
2024-12-16T00:13:21,947 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-12-16T00:13:21,972 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=c7b798bc69dfc2fada33fccf8ca28883, ASSIGN}]
2024-12-16T00:13:21,973 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=c7b798bc69dfc2fada33fccf8ca28883, ASSIGN
2024-12-16T00:13:21,974 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=c7b798bc69dfc2fada33fccf8ca28883, ASSIGN; state=OFFLINE, location=b878af803003,39441,1734308000099; forceNewPlan=false, retain=false
2024-12-16T00:13:22,021 INFO  [RS:1;b878af803003:36405 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C36405%2C1734308001768, suffix=, logDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,36405,1734308001768, archiveDir=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/oldWALs, maxLogs=32
2024-12-16T00:13:22,022 INFO  [RS:1;b878af803003:36405 {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C36405%2C1734308001768.1734308002021
2024-12-16T00:13:22,028 INFO  [RS:1;b878af803003:36405 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,36405,1734308001768/b878af803003%2C36405%2C1734308001768.1734308002021
2024-12-16T00:13:22,029 DEBUG [RS:1;b878af803003:36405 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36599:36599),(127.0.0.1/127.0.0.1:41127:41127)]
2024-12-16T00:13:22,128 INFO  [b878af803003:39029 {}] balancer.BaseLoadBalancer(546): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-16T00:13:22,129 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=c7b798bc69dfc2fada33fccf8ca28883, regionState=OPENING, regionLocation=b878af803003,39441,1734308000099
2024-12-16T00:13:22,133 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure c7b798bc69dfc2fada33fccf8ca28883, server=b878af803003,39441,1734308000099}]
2024-12-16T00:13:22,288 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to b878af803003,39441,1734308000099
2024-12-16T00:13:22,297 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:13:22,298 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => c7b798bc69dfc2fada33fccf8ca28883, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:13:22,299 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath c7b798bc69dfc2fada33fccf8ca28883
2024-12-16T00:13:22,299 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:13:22,299 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for c7b798bc69dfc2fada33fccf8ca28883
2024-12-16T00:13:22,300 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for c7b798bc69dfc2fada33fccf8ca28883
2024-12-16T00:13:22,302 INFO  [StoreOpener-c7b798bc69dfc2fada33fccf8ca28883-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c7b798bc69dfc2fada33fccf8ca28883 
2024-12-16T00:13:22,304 INFO  [StoreOpener-c7b798bc69dfc2fada33fccf8ca28883-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7b798bc69dfc2fada33fccf8ca28883 columnFamilyName info
2024-12-16T00:13:22,304 DEBUG [StoreOpener-c7b798bc69dfc2fada33fccf8ca28883-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:13:22,305 INFO  [StoreOpener-c7b798bc69dfc2fada33fccf8ca28883-1 {}] regionserver.HStore(327): Store=c7b798bc69dfc2fada33fccf8ca28883/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:13:22,306 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883
2024-12-16T00:13:22,306 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883
2024-12-16T00:13:22,309 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for c7b798bc69dfc2fada33fccf8ca28883
2024-12-16T00:13:22,311 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:13:22,312 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened c7b798bc69dfc2fada33fccf8ca28883; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=816202, jitterRate=0.037854716181755066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:13:22,313 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for c7b798bc69dfc2fada33fccf8ca28883:

2024-12-16T00:13:22,314 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883., pid=11, masterSystemTime=1734308002287
2024-12-16T00:13:22,316 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:13:22,316 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:13:22,317 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=c7b798bc69dfc2fada33fccf8ca28883, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,39441,1734308000099
2024-12-16T00:13:22,321 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-16T00:13:22,322 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure c7b798bc69dfc2fada33fccf8ca28883, server=b878af803003,39441,1734308000099 in 186 msec
2024-12-16T00:13:22,323 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-16T00:13:22,324 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=c7b798bc69dfc2fada33fccf8ca28883, ASSIGN in 350 msec
2024-12-16T00:13:22,324 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-16T00:13:22,325 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308002324"}]},"ts":"1734308002324"}
2024-12-16T00:13:22,326 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-12-16T00:13:22,331 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-12-16T00:13:22,333 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 411 msec
2024-12-16T00:13:22,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:13:22,768 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:13:23,284 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-16T00:13:23,290 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:13:23,315 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:13:26,485 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-16T00:13:26,488 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-16T00:13:26,489 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-12-16T00:13:26,862 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-16T00:13:26,863 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-12-16T00:13:26,866 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-16T00:13:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39029 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-16T00:13:31,927 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath, procId: 9 completed
2024-12-16T00:13:31,932 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-12-16T00:13:31,932 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:13:31,946 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-16T00:13:31,948 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:13:31,969 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:13:31,978 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:13:31,981 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:13:31,982 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:13:31,982 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:13:31,982 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-16T00:13:31,983 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e294bcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:13:31,983 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@55fbf7db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:13:32,073 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@822a2d9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/java.io.tmpdir/jetty-localhost-42433-hadoop-hdfs-3_4_1-tests_jar-_-any-18299073773366957545/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:32,073 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@cdf6421{HTTP/1.1, (http/1.1)}{localhost:42433}
2024-12-16T00:13:32,073 INFO  [Time-limited test {}] server.Server(415): Started @130738ms
2024-12-16T00:13:32,075 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:13:32,105 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:13:32,110 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:13:32,110 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:13:32,110 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:13:32,111 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-16T00:13:32,111 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d1550cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:13:32,111 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a962a40{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:13:32,202 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65d06642{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/java.io.tmpdir/jetty-localhost-36509-hadoop-hdfs-3_4_1-tests_jar-_-any-4317909764384341007/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:32,202 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1ee23b09{HTTP/1.1, (http/1.1)}{localhost:36509}
2024-12-16T00:13:32,202 INFO  [Time-limited test {}] server.Server(415): Started @130867ms
2024-12-16T00:13:32,204 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:13:32,229 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:13:32,232 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:13:32,233 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:13:32,233 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:13:32,233 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:13:32,234 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fb180fe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:13:32,234 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58d3f3f2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:13:32,325 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@280164fa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/java.io.tmpdir/jetty-localhost-43351-hadoop-hdfs-3_4_1-tests_jar-_-any-11995571507558623604/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:32,325 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1e996930{HTTP/1.1, (http/1.1)}{localhost:43351}
2024-12-16T00:13:32,325 INFO  [Time-limited test {}] server.Server(415): Started @130990ms
2024-12-16T00:13:32,327 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:13:33,163 WARN  [Thread-675 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data5/current/BP-1525461463-172.17.0.2-1734307998365/current, will proceed with Du for space computation calculation, 
2024-12-16T00:13:33,163 WARN  [Thread-676 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data6/current/BP-1525461463-172.17.0.2-1734307998365/current, will proceed with Du for space computation calculation, 
2024-12-16T00:13:33,181 WARN  [Thread-616 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:13:33,183 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7f06f49d1801dc41 with lease ID 0x125ba5608a7cc50b: Processing first storage report for DS-ae04eca9-a881-4228-b6b9-85ae9948933b from datanode DatanodeRegistration(127.0.0.1:46225, datanodeUuid=6c965a38-0551-4c3d-a47f-dfa4cf6d089f, infoPort=39777, infoSecurePort=0, ipcPort=43477, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365)
2024-12-16T00:13:33,183 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7f06f49d1801dc41 with lease ID 0x125ba5608a7cc50b: from storage DS-ae04eca9-a881-4228-b6b9-85ae9948933b node DatanodeRegistration(127.0.0.1:46225, datanodeUuid=6c965a38-0551-4c3d-a47f-dfa4cf6d089f, infoPort=39777, infoSecurePort=0, ipcPort=43477, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:13:33,183 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7f06f49d1801dc41 with lease ID 0x125ba5608a7cc50b: Processing first storage report for DS-438891f6-ee91-4369-aeb4-df6bb28e809e from datanode DatanodeRegistration(127.0.0.1:46225, datanodeUuid=6c965a38-0551-4c3d-a47f-dfa4cf6d089f, infoPort=39777, infoSecurePort=0, ipcPort=43477, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365)
2024-12-16T00:13:33,183 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7f06f49d1801dc41 with lease ID 0x125ba5608a7cc50b: from storage DS-438891f6-ee91-4369-aeb4-df6bb28e809e node DatanodeRegistration(127.0.0.1:46225, datanodeUuid=6c965a38-0551-4c3d-a47f-dfa4cf6d089f, infoPort=39777, infoSecurePort=0, ipcPort=43477, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:13:33,225 WARN  [Thread-686 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data7/current/BP-1525461463-172.17.0.2-1734307998365/current, will proceed with Du for space computation calculation, 
2024-12-16T00:13:33,225 WARN  [Thread-687 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data8/current/BP-1525461463-172.17.0.2-1734307998365/current, will proceed with Du for space computation calculation, 
2024-12-16T00:13:33,241 WARN  [Thread-638 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:13:33,243 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6c806219397f65b4 with lease ID 0x125ba5608a7cc50c: Processing first storage report for DS-075199ab-2554-4f59-837f-37282af91993 from datanode DatanodeRegistration(127.0.0.1:33583, datanodeUuid=5213e0d5-9d47-47f0-806f-daabc8822390, infoPort=43449, infoSecurePort=0, ipcPort=33643, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365)
2024-12-16T00:13:33,243 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6c806219397f65b4 with lease ID 0x125ba5608a7cc50c: from storage DS-075199ab-2554-4f59-837f-37282af91993 node DatanodeRegistration(127.0.0.1:33583, datanodeUuid=5213e0d5-9d47-47f0-806f-daabc8822390, infoPort=43449, infoSecurePort=0, ipcPort=33643, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:13:33,243 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6c806219397f65b4 with lease ID 0x125ba5608a7cc50c: Processing first storage report for DS-d27155e9-376c-4e59-83a0-c2d346a0b968 from datanode DatanodeRegistration(127.0.0.1:33583, datanodeUuid=5213e0d5-9d47-47f0-806f-daabc8822390, infoPort=43449, infoSecurePort=0, ipcPort=33643, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365)
2024-12-16T00:13:33,244 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6c806219397f65b4 with lease ID 0x125ba5608a7cc50c: from storage DS-d27155e9-376c-4e59-83a0-c2d346a0b968 node DatanodeRegistration(127.0.0.1:33583, datanodeUuid=5213e0d5-9d47-47f0-806f-daabc8822390, infoPort=43449, infoSecurePort=0, ipcPort=33643, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:13:33,258 WARN  [Thread-697 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9/current/BP-1525461463-172.17.0.2-1734307998365/current, will proceed with Du for space computation calculation, 
2024-12-16T00:13:33,258 WARN  [Thread-698 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10/current/BP-1525461463-172.17.0.2-1734307998365/current, will proceed with Du for space computation calculation, 
2024-12-16T00:13:33,275 WARN  [Thread-660 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:13:33,278 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd30f00b06d1efa0e with lease ID 0x125ba5608a7cc50d: Processing first storage report for DS-64c58a24-5d97-4ae2-b99c-46053965fe45 from datanode DatanodeRegistration(127.0.0.1:44051, datanodeUuid=e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce, infoPort=33049, infoSecurePort=0, ipcPort=33223, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365)
2024-12-16T00:13:33,279 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd30f00b06d1efa0e with lease ID 0x125ba5608a7cc50d: from storage DS-64c58a24-5d97-4ae2-b99c-46053965fe45 node DatanodeRegistration(127.0.0.1:44051, datanodeUuid=e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce, infoPort=33049, infoSecurePort=0, ipcPort=33223, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:13:33,279 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd30f00b06d1efa0e with lease ID 0x125ba5608a7cc50d: Processing first storage report for DS-b9a71f33-1576-473b-94af-e3d9f9f4e4ea from datanode DatanodeRegistration(127.0.0.1:44051, datanodeUuid=e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce, infoPort=33049, infoSecurePort=0, ipcPort=33223, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365)
2024-12-16T00:13:33,279 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd30f00b06d1efa0e with lease ID 0x125ba5608a7cc50d: from storage DS-b9a71f33-1576-473b-94af-e3d9f9f4e4ea node DatanodeRegistration(127.0.0.1:44051, datanodeUuid=e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce, infoPort=33049, infoSecurePort=0, ipcPort=33223, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:13:33,368 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7a270e7d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:33,368 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@75ee43d4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:13:33,368 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:13:33,368 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48cc4a03{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:13:33,368 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ed21f9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,STOPPED}
2024-12-16T00:13:33,370 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:13:33,370 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:13:33,370 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1525461463-172.17.0.2-1734307998365 (Datanode Uuid ed51c821-41de-47d1-8ef3-1a57dee79fe3) service to localhost/127.0.0.1:37443
2024-12-16T00:13:33,370 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:13:33,365 WARN  [ResponseProcessor for block BP-1525461463-172.17.0.2-1734307998365:blk_1073741839_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1525461463-172.17.0.2-1734307998365:blk_1073741839_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,365 WARN  [ResponseProcessor for block BP-1525461463-172.17.0.2-1734307998365:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1525461463-172.17.0.2-1734307998365:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,371 WARN  [DataStreamer for file /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta block BP-1525461463-172.17.0.2-1734307998365:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK], DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]) is bad.
2024-12-16T00:13:33,371 WARN  [DataStreamer for file /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,36405,1734308001768/b878af803003%2C36405%2C1734308001768.1734308002021 block BP-1525461463-172.17.0.2-1734307998365:blk_1073741839_1015 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741839_1015 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK], DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]) is bad.
2024-12-16T00:13:33,371 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data3/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:33,371 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data4/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:33,371 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:41826 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43815:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41826 dst: /127.0.0.1:43815
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:33,371 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-386359270_22 at /127.0.0.1:49616 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:38223:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49616 dst: /127.0.0.1:38223
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 48963 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:33,365 WARN  [ResponseProcessor for block BP-1525461463-172.17.0.2-1734307998365:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1525461463-172.17.0.2-1734307998365:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-1525461463-172.17.0.2-1734307998365:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,373 WARN  [DataStreamer for file /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308000230 block BP-1525461463-172.17.0.2-1734307998365:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK], DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]) is bad.
2024-12-16T00:13:33,371 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965178947_22 at /127.0.0.1:41884 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:43815:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41884 dst: /127.0.0.1:43815
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:33,372 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:49656 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:38223:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49656 dst: /127.0.0.1:38223
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 48957 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:33,371 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:49644 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:38223:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49644 dst: /127.0.0.1:38223
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 48945 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:33,371 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1965178947_22 at /127.0.0.1:49710 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:38223:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49710 dst: /127.0.0.1:38223
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 48658 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:33,373 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:41834 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43815:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41834 dst: /127.0.0.1:43815
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:33,373 WARN  [PacketResponder: BP-1525461463-172.17.0.2-1734307998365:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:38223] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:33,374 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-386359270_22 at /127.0.0.1:41788 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43815:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41788 dst: /127.0.0.1:43815
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:33,374 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:13:33,365 WARN  [ResponseProcessor for block BP-1525461463-172.17.0.2-1734307998365:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1525461463-172.17.0.2-1734307998365:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,374 WARN  [DataStreamer for file /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 block BP-1525461463-172.17.0.2-1734307998365:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK], DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]) is bad.
2024-12-16T00:13:33,377 WARN  [DataStreamer for file /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308000230 block BP-1525461463-172.17.0.2-1734307998365:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,377 WARN  [DataStreamer for file /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 block BP-1525461463-172.17.0.2-1734307998365:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,377 WARN  [DataStreamer for file /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta block BP-1525461463-172.17.0.2-1734307998365:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,378 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3ca4a1d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:33,378 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4dfe23a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:13:33,378 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:13:33,378 WARN  [DataStreamer for file /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,36405,1734308001768/b878af803003%2C36405%2C1734308001768.1734308002021 block BP-1525461463-172.17.0.2-1734307998365:blk_1073741839_1015 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,379 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f590214{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:13:33,379 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b2f8217{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,STOPPED}
2024-12-16T00:13:33,380 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:13:33,380 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:13:33,380 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:13:33,380 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1525461463-172.17.0.2-1734307998365 (Datanode Uuid fdbe45cc-ce4f-48e3-bbfe-26744033b547) service to localhost/127.0.0.1:37443
2024-12-16T00:13:33,381 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data1/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:33,381 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data2/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:33,381 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:13:33,384 WARN  [RS:0;b878af803003:39441.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=4, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,385 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C39441%2C1734308000099:(num 1734308000629) roll requested
2024-12-16T00:13:33,385 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C39441%2C1734308000099.1734308013385
2024-12-16T00:13:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39441 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39441 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:43612 deadline: 1734308023384, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL
2024-12-16T00:13:33,388 WARN  [Thread-708 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1020
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,388 WARN  [Thread-708 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741840_1020 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK], DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]) is bad.
2024-12-16T00:13:33,389 WARN  [Thread-708 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741840_1020
2024-12-16T00:13:33,391 WARN  [Thread-708 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]
2024-12-16T00:13:33,393 WARN  [Thread-708 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1021
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,394 WARN  [Thread-708 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741841_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK], DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]) is bad.
2024-12-16T00:13:33,394 WARN  [Thread-708 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741841_1021
2024-12-16T00:13:33,394 WARN  [Thread-708 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]
2024-12-16T00:13:33,400 WARN  [regionserver/b878af803003:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL
2024-12-16T00:13:33,400 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 with entries=4, filesize=959 B; new WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308013385
2024-12-16T00:13:33,401 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43449:43449),(127.0.0.1/127.0.0.1:39777:39777)]
2024-12-16T00:13:33,401 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 is not closed yet, will try archiving it next time
2024-12-16T00:13:33,401 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,401 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:33,402 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-12-16T00:13:33,402 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-12-16T00:13:33,402 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629
2024-12-16T00:13:33,404 WARN  [IPC Server handler 4 on default port 37443 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 has not been closed. Lease recovery is in progress. RecoveryId = 1023 for block blk_1073741833_1009
2024-12-16T00:13:33,406 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 after 3ms
2024-12-16T00:13:37,408 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 after 4006ms
2024-12-16T00:13:45,488 INFO  [Time-limited test {}] wal.TestLogRolling(243): log.getCurrentFileName(): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308013385
2024-12-16T00:13:45,489 WARN  [ResponseProcessor for block BP-1525461463-172.17.0.2-1734307998365:blk_1073741842_1022 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1525461463-172.17.0.2-1734307998365:blk_1073741842_1022
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:45,489 WARN  [DataStreamer for file /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308013385 block BP-1525461463-172.17.0.2-1734307998365:blk_1073741842_1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741842_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK], DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]) is bad.
2024-12-16T00:13:45,490 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:46926 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741842_1022] {}] datanode.DataXceiver(331): 127.0.0.1:33583:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46926 dst: /127.0.0.1:33583
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:45,491 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:55856 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741842_1022] {}] datanode.DataXceiver(331): 127.0.0.1:46225:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55856 dst: /127.0.0.1:46225
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:45,540 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@65d06642{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:45,541 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1ee23b09{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:13:45,541 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:13:45,541 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a962a40{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:13:45,541 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d1550cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,STOPPED}
2024-12-16T00:13:45,544 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:13:45,544 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:13:45,544 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:13:45,544 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1525461463-172.17.0.2-1734307998365 (Datanode Uuid 5213e0d5-9d47-47f0-806f-daabc8822390) service to localhost/127.0.0.1:37443
2024-12-16T00:13:45,545 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data7/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:45,545 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data8/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:45,545 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:13:45,548 WARN  [sync.1 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]]
2024-12-16T00:13:45,549 WARN  [sync.1 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]]
2024-12-16T00:13:45,549 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C39441%2C1734308000099:(num 1734308013385) roll requested
2024-12-16T00:13:45,549 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C39441%2C1734308000099.1734308025549
2024-12-16T00:13:45,557 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308013385 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308025549
2024-12-16T00:13:45,558 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39777:39777),(127.0.0.1/127.0.0.1:33049:33049)]
2024-12-16T00:13:45,558 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 is not closed yet, will try archiving it next time
2024-12-16T00:13:45,558 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308013385 is not closed yet, will try archiving it next time
2024-12-16T00:13:45,560 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46225 is added to blk_1073741842_1024 (size=2431)
2024-12-16T00:13:45,962 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 is not closed yet, will try archiving it next time
2024-12-16T00:13:46,199 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@600c5d8d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46225, datanodeUuid=6c965a38-0551-4c3d-a47f-dfa4cf6d089f, infoPort=39777, infoSecurePort=0, ipcPort=43477, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741842_1024 to 127.0.0.1:38223 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,554 WARN  [ResponseProcessor for block BP-1525461463-172.17.0.2-1734307998365:blk_1073741843_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1525461463-172.17.0.2-1734307998365:blk_1073741843_1025
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,555 WARN  [DataStreamer for file /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308025549 block BP-1525461463-172.17.0.2-1734307998365:blk_1073741843_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK], DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]) is bad.
2024-12-16T00:13:49,556 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:55596 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:46225:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55596 dst: /127.0.0.1:46225
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,557 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57038 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:44051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57038 dst: /127.0.0.1:44051
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,578 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@822a2d9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:49,579 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@cdf6421{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:13:49,579 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:13:49,579 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@55fbf7db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:13:49,579 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e294bcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,STOPPED}
2024-12-16T00:13:49,581 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:13:49,581 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:13:49,581 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1525461463-172.17.0.2-1734307998365 (Datanode Uuid 6c965a38-0551-4c3d-a47f-dfa4cf6d089f) service to localhost/127.0.0.1:37443
2024-12-16T00:13:49,581 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:13:49,582 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data5/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:49,582 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data6/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:13:49,583 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:13:49,585 WARN  [sync.4 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]
2024-12-16T00:13:49,585 WARN  [sync.4 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]
2024-12-16T00:13:49,585 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C39441%2C1734308000099:(num 1734308025549) roll requested
2024-12-16T00:13:49,586 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C39441%2C1734308000099.1734308029586
2024-12-16T00:13:49,589 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,590 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK], DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]) is bad.
2024-12-16T00:13:49,590 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741844_1027
2024-12-16T00:13:49,591 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]
2024-12-16T00:13:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39441 {}] regionserver.HRegion(8581): Flush requested on c7b798bc69dfc2fada33fccf8ca28883
2024-12-16T00:13:49,591 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing c7b798bc69dfc2fada33fccf8ca28883 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-16T00:13:49,594 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57058 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741845_1028] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10]'}, localName='127.0.0.1:44051', datanodeUuid='e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce', xmitsInProgress=0}:Exception transferring block BP-1525461463-172.17.0.2-1734307998365:blk_1073741845_1028 to mirror 127.0.0.1:43815
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,595 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57058 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741845_1028] {}] datanode.BlockReceiver(316): Block 1073741845 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-16T00:13:49,595 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43815
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,595 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK], DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]) is bad.
2024-12-16T00:13:49,596 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741845_1028
2024-12-16T00:13:49,595 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57058 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741845_1028] {}] datanode.DataXceiver(331): 127.0.0.1:44051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57058 dst: /127.0.0.1:44051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,596 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]
2024-12-16T00:13:49,599 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46225
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,599 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57072 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10]'}, localName='127.0.0.1:44051', datanodeUuid='e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce', xmitsInProgress=0}:Exception transferring block BP-1525461463-172.17.0.2-1734307998365:blk_1073741846_1029 to mirror 127.0.0.1:46225
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,599 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK], DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]) is bad.
2024-12-16T00:13:49,599 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57072 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-16T00:13:49,599 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741846_1029
2024-12-16T00:13:49,600 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57072 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:44051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57072 dst: /127.0.0.1:44051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,600 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]
2024-12-16T00:13:49,604 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38223
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,604 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57080 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741847_1030] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10]'}, localName='127.0.0.1:44051', datanodeUuid='e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce', xmitsInProgress=0}:Exception transferring block BP-1525461463-172.17.0.2-1734307998365:blk_1073741847_1030 to mirror 127.0.0.1:38223
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,604 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK], DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]) is bad.
2024-12-16T00:13:49,604 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741847_1030
2024-12-16T00:13:49,604 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57080 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741847_1030] {}] datanode.BlockReceiver(316): Block 1073741847 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-16T00:13:49,604 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57080 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741847_1030] {}] datanode.DataXceiver(331): 127.0.0.1:44051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57080 dst: /127.0.0.1:44051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,606 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]
2024-12-16T00:13:49,607 WARN  [IPC Server handler 3 on default port 37443 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-16T00:13:49,607 WARN  [IPC Server handler 3 on default port 37443 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-16T00:13:49,607 WARN  [IPC Server handler 3 on default port 37443 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-16T00:13:49,611 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308025549 with entries=13, filesize=14.10 KB; new WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308029586
2024-12-16T00:13:49,611 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33049:33049)]
2024-12-16T00:13:49,611 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 is not closed yet, will try archiving it next time
2024-12-16T00:13:49,611 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308025549 is not closed yet, will try archiving it next time
2024-12-16T00:13:49,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/.tmp/info/df52de0a826d4bbb9ea7674309202e86 is 1080, key is row0002/info:/1734308025546/Put/seqid=0
2024-12-16T00:13:49,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741843_1026 (size=14443)
2024-12-16T00:13:49,613 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,613 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK], DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]) is bad.
2024-12-16T00:13:49,613 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741849_1032
2024-12-16T00:13:49,614 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]
2024-12-16T00:13:49,615 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,615 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK], DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]) is bad.
2024-12-16T00:13:49,615 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741850_1033
2024-12-16T00:13:49,616 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]
2024-12-16T00:13:49,617 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,617 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK], DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]) is bad.
2024-12-16T00:13:49,617 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741851_1034
2024-12-16T00:13:49,618 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]
2024-12-16T00:13:49,620 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33583
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,620 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57100 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741852_1035] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10]'}, localName='127.0.0.1:44051', datanodeUuid='e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce', xmitsInProgress=0}:Exception transferring block BP-1525461463-172.17.0.2-1734307998365:blk_1073741852_1035 to mirror 127.0.0.1:33583
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,620 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK], DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]) is bad.
2024-12-16T00:13:49,620 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741852_1035
2024-12-16T00:13:49,620 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57100 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741852_1035] {}] datanode.BlockReceiver(316): Block 1073741852 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-16T00:13:49,620 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57100 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741852_1035] {}] datanode.DataXceiver(331): 127.0.0.1:44051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57100 dst: /127.0.0.1:44051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,621 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]
2024-12-16T00:13:49,621 WARN  [IPC Server handler 0 on default port 37443 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-16T00:13:49,622 WARN  [IPC Server handler 0 on default port 37443 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-16T00:13:49,622 WARN  [IPC Server handler 0 on default port 37443 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-16T00:13:49,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741853_1036 (size=10347)
2024-12-16T00:13:49,809 WARN  [sync.2 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]
2024-12-16T00:13:49,809 WARN  [sync.2 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]
2024-12-16T00:13:49,810 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C39441%2C1734308000099:(num 1734308029586) roll requested
2024-12-16T00:13:49,810 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C39441%2C1734308000099.1734308029810
2024-12-16T00:13:49,817 WARN  [Thread-739 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,817 WARN  [Thread-739 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK], DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]) is bad.
2024-12-16T00:13:49,817 WARN  [Thread-739 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741854_1037
2024-12-16T00:13:49,819 WARN  [Thread-739 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]
2024-12-16T00:13:49,824 WARN  [Thread-739 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46225
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,824 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57130 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741855_1038] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10]'}, localName='127.0.0.1:44051', datanodeUuid='e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce', xmitsInProgress=0}:Exception transferring block BP-1525461463-172.17.0.2-1734307998365:blk_1073741855_1038 to mirror 127.0.0.1:46225
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,825 WARN  [Thread-739 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK], DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]) is bad.
2024-12-16T00:13:49,825 WARN  [Thread-739 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741855_1038
2024-12-16T00:13:49,825 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57130 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741855_1038] {}] datanode.BlockReceiver(316): Block 1073741855 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-16T00:13:49,825 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57130 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741855_1038] {}] datanode.DataXceiver(331): 127.0.0.1:44051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57130 dst: /127.0.0.1:44051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,826 WARN  [Thread-739 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]
2024-12-16T00:13:49,827 WARN  [Thread-739 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,827 WARN  [Thread-739 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK], DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]) is bad.
2024-12-16T00:13:49,827 WARN  [Thread-739 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741856_1039
2024-12-16T00:13:49,828 WARN  [Thread-739 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]
2024-12-16T00:13:49,830 WARN  [Thread-739 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38223
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:49,831 WARN  [Thread-739 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK], DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]) is bad.
2024-12-16T00:13:49,830 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57142 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10]'}, localName='127.0.0.1:44051', datanodeUuid='e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce', xmitsInProgress=0}:Exception transferring block BP-1525461463-172.17.0.2-1734307998365:blk_1073741857_1040 to mirror 127.0.0.1:38223
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,831 WARN  [Thread-739 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741857_1040
2024-12-16T00:13:49,831 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57142 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-16T00:13:49,831 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:57142 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:44051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57142 dst: /127.0.0.1:44051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:49,831 WARN  [Thread-739 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]
2024-12-16T00:13:49,832 WARN  [IPC Server handler 4 on default port 37443 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-16T00:13:49,832 WARN  [IPC Server handler 4 on default port 37443 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-16T00:13:49,832 WARN  [IPC Server handler 4 on default port 37443 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-16T00:13:49,836 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308029586 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308029810
2024-12-16T00:13:49,837 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33049:33049)]
2024-12-16T00:13:49,837 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 is not closed yet, will try archiving it next time
2024-12-16T00:13:49,837 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308025549 is not closed yet, will try archiving it next time
2024-12-16T00:13:49,837 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308029586 is not closed yet, will try archiving it next time
2024-12-16T00:13:49,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741848_1031 (size=1261)
2024-12-16T00:13:49,943 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-16T00:13:50,011 WARN  [sync.4 {}] wal.FSHLog(760): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-12-16T00:13:50,013 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 is not closed yet, will try archiving it next time
2024-12-16T00:13:50,013 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308029586 is not closed yet, will try archiving it next time
2024-12-16T00:13:50,026 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/.tmp/info/df52de0a826d4bbb9ea7674309202e86
2024-12-16T00:13:50,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/.tmp/info/df52de0a826d4bbb9ea7674309202e86 as hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/info/df52de0a826d4bbb9ea7674309202e86
2024-12-16T00:13:50,040 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/info/df52de0a826d4bbb9ea7674309202e86, entries=5, sequenceid=12, filesize=10.1 K
2024-12-16T00:13:50,041 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for c7b798bc69dfc2fada33fccf8ca28883 in 450ms, sequenceid=12, compaction requested=false
2024-12-16T00:13:50,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for c7b798bc69dfc2fada33fccf8ca28883:

2024-12-16T00:13:50,222 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:13:50,226 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:13:50,227 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:13:50,227 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:13:50,227 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:13:50,228 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78252885{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:13:50,228 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d5fd30b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:13:50,239 DEBUG [Close-WAL-Writer-2 {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308000629 is not closed yet, will try archiving it next time
2024-12-16T00:13:50,240 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308013385 to hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/oldWALs/b878af803003%2C39441%2C1734308000099.1734308013385
2024-12-16T00:13:50,320 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d2e9057{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/java.io.tmpdir/jetty-localhost-40381-hadoop-hdfs-3_4_1-tests_jar-_-any-14956421731648158338/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:13:50,320 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3ce3e309{HTTP/1.1, (http/1.1)}{localhost:40381}
2024-12-16T00:13:50,320 INFO  [Time-limited test {}] server.Server(415): Started @148985ms
2024-12-16T00:13:50,322 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:13:50,414 WARN  [master/b878af803003:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=96, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:50,414 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C39029%2C1734307999962:(num 1734308000230) roll requested
2024-12-16T00:13:50,415 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C39029%2C1734307999962.1734308030414
2024-12-16T00:13:50,415 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:50,415 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
	at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:50,419 WARN  [Thread-767 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:50,419 WARN  [Thread-767 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK], DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]) is bad.
2024-12-16T00:13:50,419 WARN  [Thread-767 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741859_1042
2024-12-16T00:13:50,420 WARN  [Thread-767 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]
2024-12-16T00:13:50,424 WARN  [Thread-767 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38223
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:50,424 WARN  [Thread-767 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK], DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]) is bad.
2024-12-16T00:13:50,424 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-386359270_22 at /127.0.0.1:35136 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10]'}, localName='127.0.0.1:44051', datanodeUuid='e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce', xmitsInProgress=0}:Exception transferring block BP-1525461463-172.17.0.2-1734307998365:blk_1073741860_1043 to mirror 127.0.0.1:38223
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:50,424 WARN  [Thread-767 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741860_1043
2024-12-16T00:13:50,424 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-386359270_22 at /127.0.0.1:35136 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-16T00:13:50,424 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-386359270_22 at /127.0.0.1:35136 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:44051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35136 dst: /127.0.0.1:44051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:50,425 WARN  [Thread-767 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38223,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]
2024-12-16T00:13:50,427 WARN  [Thread-767 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46225
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:50,427 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-386359270_22 at /127.0.0.1:35150 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741861_1044] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10]'}, localName='127.0.0.1:44051', datanodeUuid='e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce', xmitsInProgress=0}:Exception transferring block BP-1525461463-172.17.0.2-1734307998365:blk_1073741861_1044 to mirror 127.0.0.1:46225
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:50,427 WARN  [Thread-767 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK], DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]) is bad.
2024-12-16T00:13:50,427 WARN  [Thread-767 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741861_1044
2024-12-16T00:13:50,427 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-386359270_22 at /127.0.0.1:35150 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741861_1044] {}] datanode.BlockReceiver(316): Block 1073741861 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-16T00:13:50,427 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-386359270_22 at /127.0.0.1:35150 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741861_1044] {}] datanode.DataXceiver(331): 127.0.0.1:44051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35150 dst: /127.0.0.1:44051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:50,428 WARN  [Thread-767 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]
2024-12-16T00:13:50,430 WARN  [Thread-767 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43815
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:50,430 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-386359270_22 at /127.0.0.1:35160 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10]'}, localName='127.0.0.1:44051', datanodeUuid='e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce', xmitsInProgress=0}:Exception transferring block BP-1525461463-172.17.0.2-1734307998365:blk_1073741862_1045 to mirror 127.0.0.1:43815
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:50,430 WARN  [Thread-767 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK], DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]) is bad.
2024-12-16T00:13:50,430 WARN  [Thread-767 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741862_1045
2024-12-16T00:13:50,431 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-386359270_22 at /127.0.0.1:35160 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-16T00:13:50,431 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-386359270_22 at /127.0.0.1:35160 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:44051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35160 dst: /127.0.0.1:44051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:50,431 WARN  [Thread-767 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]
2024-12-16T00:13:50,432 WARN  [IPC Server handler 2 on default port 37443 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-16T00:13:50,432 WARN  [IPC Server handler 2 on default port 37443 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-16T00:13:50,432 WARN  [IPC Server handler 2 on default port 37443 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-16T00:13:50,435 WARN  [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
2024-12-16T00:13:50,435 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308000230 with entries=93, filesize=46.05 KB; new WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308030414
2024-12-16T00:13:50,435 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33049:33049)]
2024-12-16T00:13:50,435 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308000230 is not closed yet, will try archiving it next time
2024-12-16T00:13:50,435 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:50,435 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:13:50,436 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308000230
2024-12-16T00:13:50,436 WARN  [IPC Server handler 4 on default port 37443 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308000230 has not been closed. Lease recovery is in progress. RecoveryId = 1047 for block blk_1073741830_1006
2024-12-16T00:13:50,436 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308000230 after 0ms
2024-12-16T00:13:50,695 WARN  [Thread-761 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:13:50,702 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5f74a5f1984f55d1 with lease ID 0x125ba5608a7cc50e: from storage DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5 node DatanodeRegistration(127.0.0.1:41131, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=34267, infoSecurePort=0, ipcPort=44475, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:13:50,702 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5f74a5f1984f55d1 with lease ID 0x125ba5608a7cc50e: from storage DS-02cd5796-a69e-49e8-9424-2062466f8ed7 node DatanodeRegistration(127.0.0.1:41131, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=34267, infoSecurePort=0, ipcPort=44475, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:13:52,283 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4d8067b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44051, datanodeUuid=e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce, infoPort=33049, infoSecurePort=0, ipcPort=33223, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741853_1036 to 127.0.0.1:33583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:52,285 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41131 is added to blk_1073741843_1026 (size=14443)
2024-12-16T00:13:53,284 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@a1ff948[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44051, datanodeUuid=e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce, infoPort=33049, infoSecurePort=0, ipcPort=33223, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741848_1031 to 127.0.0.1:46225 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:13:53,815 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:13:53,817 INFO  [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53578, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:13:54,439 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308000230 after 4003ms
2024-12-16T00:13:55,465 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:13:55,468 INFO  [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53584, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:14:00,718 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@5cc315f1 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1525461463-172.17.0.2-1734307998365:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:43815,null,null])
java.net.ConnectException: Call From b878af803003/172.17.0.2 to localhost:38427 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-16T00:14:00,724 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41131 is added to blk_1073741833_1023 (size=959)
2024-12-16T00:14:01,704 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741837_1013 (size=393)
2024-12-16T00:14:01,705 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:14:02,338 INFO  [master/b878af803003:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-16T00:14:02,338 INFO  [master/b878af803003:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-16T00:14:02,702 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@38a2af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41131, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=34267, infoSecurePort=0, ipcPort=44475, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741831_1007 to 127.0.0.1:46225 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:02,702 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@6dd5ed2d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41131, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=34267, infoSecurePort=0, ipcPort=44475, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741829_1005 to 127.0.0.1:46225 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:04,700 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@6dd5ed2d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41131, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=34267, infoSecurePort=0, ipcPort=44475, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741832_1008 to 127.0.0.1:33583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:04,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:14:05,703 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@38a2af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41131, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=34267, infoSecurePort=0, ipcPort=44475, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741826_1002 to 127.0.0.1:33583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:05,703 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@6dd5ed2d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41131, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=34267, infoSecurePort=0, ipcPort=44475, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741833_1023 to 127.0.0.1:46225 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:07,299 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region c7b798bc69dfc2fada33fccf8ca28883, had cached 0 bytes from a total of 10347
2024-12-16T00:14:07,702 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@38a2af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41131, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=34267, infoSecurePort=0, ipcPort=44475, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741825_1001 to 127.0.0.1:46225 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:07,707 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:14:08,700 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@38a2af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41131, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=34267, infoSecurePort=0, ipcPort=44475, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741836_1012 to 127.0.0.1:46225 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:08,700 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@6dd5ed2d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41131, datanodeUuid=ed51c821-41de-47d1-8ef3-1a57dee79fe3, infoPort=34267, infoSecurePort=0, ipcPort=44475, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741838_1014 to 127.0.0.1:33583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:09,250 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C39441%2C1734308000099.1734308049249
2024-12-16T00:14:09,262 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308029810 with entries=2, filesize=1.57 KB; new WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308049249
2024-12-16T00:14:09,262 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34267:34267),(127.0.0.1/127.0.0.1:33049:33049)]
2024-12-16T00:14:09,262 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308029810 is not closed yet, will try archiving it next time
2024-12-16T00:14:09,264 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741858_1041 (size=1618)
2024-12-16T00:14:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39441 {}] regionserver.HRegion(8581): Flush requested on c7b798bc69dfc2fada33fccf8ca28883
2024-12-16T00:14:09,265 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing c7b798bc69dfc2fada33fccf8ca28883 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-12-16T00:14:09,266 INFO  [sync.3 {}] wal.FSHLog(777): LowReplication-Roller was enabled.
2024-12-16T00:14:09,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/.tmp/info/299ec5e36052452885942e148aa1af7b is 1080, key is row0007/info:/1734308029592/Put/seqid=0
2024-12-16T00:14:09,273 WARN  [Thread-806 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1049
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33583
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:09,273 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:33706 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741865_1049] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data4]'}, localName='127.0.0.1:41131', datanodeUuid='ed51c821-41de-47d1-8ef3-1a57dee79fe3', xmitsInProgress=0}:Exception transferring block BP-1525461463-172.17.0.2-1734307998365:blk_1073741865_1049 to mirror 127.0.0.1:33583
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:09,273 WARN  [Thread-806 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741865_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41131,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK], DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]) is bad.
2024-12-16T00:14:09,274 WARN  [Thread-806 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741865_1049
2024-12-16T00:14:09,274 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:33706 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741865_1049] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-16T00:14:09,274 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:33706 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741865_1049] {}] datanode.DataXceiver(331): 127.0.0.1:41131:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33706 dst: /127.0.0.1:41131
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:09,274 WARN  [Thread-806 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]
2024-12-16T00:14:09,276 WARN  [Thread-806 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:09,276 WARN  [Thread-806 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741866_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK], DatanodeInfoWithStorage[127.0.0.1:41131,DS-bfaae6e0-e968-46be-b7ed-c4526b4685b5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]) is bad.
2024-12-16T00:14:09,276 WARN  [Thread-806 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741866_1050
2024-12-16T00:14:09,277 WARN  [Thread-806 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]
2024-12-16T00:14:09,282 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741867_1051 (size=13583)
2024-12-16T00:14:09,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41131 is added to blk_1073741867_1051 (size=13583)
2024-12-16T00:14:09,283 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=25 (bloomFilter=true), to=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/.tmp/info/299ec5e36052452885942e148aa1af7b
2024-12-16T00:14:09,283 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-16T00:14:09,283 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-16T00:14:09,283 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3cfd3e0f to 127.0.0.1:50614
2024-12-16T00:14:09,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:14:09,283 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-16T00:14:09,284 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2062255577, stopped=false
2024-12-16T00:14:09,284 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=b878af803003,39029,1734307999962
2024-12-16T00:14:09,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/.tmp/info/299ec5e36052452885942e148aa1af7b as hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/info/299ec5e36052452885942e148aa1af7b
2024-12-16T00:14:09,297 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/info/299ec5e36052452885942e148aa1af7b, entries=8, sequenceid=25, filesize=13.3 K
2024-12-16T00:14:09,298 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~10.50 KB/10757, heapSize ~11.48 KB/11760, currentSize=9.46 KB/9684 for c7b798bc69dfc2fada33fccf8ca28883 in 33ms, sequenceid=25, compaction requested=false
2024-12-16T00:14:09,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for c7b798bc69dfc2fada33fccf8ca28883:

2024-12-16T00:14:09,299 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=23.4 K, sizeToCheck=16.0 K
2024-12-16T00:14:09,299 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:14:09,299 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/info/299ec5e36052452885942e148aa1af7b because midkey is the same as first or last row
2024-12-16T00:14:09,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:14:09,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36405-0x1002c2f0d4a0003, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:14:09,338 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-16T00:14:09,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36405-0x1002c2f0d4a0003, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:09,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:09,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:14:09,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:14:09,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:09,338 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,39441,1734308000099' *****
2024-12-16T00:14:09,338 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-16T00:14:09,338 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,36405,1734308001768' *****
2024-12-16T00:14:09,338 INFO  [RS:0;b878af803003:39441 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-16T00:14:09,338 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-16T00:14:09,338 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:14:09,339 INFO  [RS:0;b878af803003:39441 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-16T00:14:09,339 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36405-0x1002c2f0d4a0003, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:14:09,339 INFO  [RS:0;b878af803003:39441 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-16T00:14:09,339 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-16T00:14:09,339 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(3579): Received CLOSE for c7b798bc69dfc2fada33fccf8ca28883
2024-12-16T00:14:09,339 INFO  [RS:1;b878af803003:36405 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-16T00:14:09,339 INFO  [RS:1;b878af803003:36405 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-16T00:14:09,339 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-16T00:14:09,339 INFO  [RS:1;b878af803003:36405 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-16T00:14:09,339 INFO  [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(1224): stopping server b878af803003,36405,1734308001768
2024-12-16T00:14:09,339 DEBUG [RS:1;b878af803003:36405 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:14:09,339 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:14:09,339 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(3579): Received CLOSE for 80495c60a5e712e96c604963e7816d5b
2024-12-16T00:14:09,339 INFO  [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(1250): stopping server b878af803003,36405,1734308001768; all regions closed.
2024-12-16T00:14:09,339 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1224): stopping server b878af803003,39441,1734308000099
2024-12-16T00:14:09,339 DEBUG [RS:0;b878af803003:39441 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:14:09,339 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c7b798bc69dfc2fada33fccf8ca28883, disabling compactions & flushes
2024-12-16T00:14:09,339 INFO  [RS:0;b878af803003:39441 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-16T00:14:09,339 INFO  [RS:0;b878af803003:39441 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-16T00:14:09,339 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:14:09,339 INFO  [RS:0;b878af803003:39441 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-16T00:14:09,339 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:14:09,340 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-16T00:14:09,340 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883. after waiting 0 ms
2024-12-16T00:14:09,340 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:14:09,340 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,36405,1734308001768
2024-12-16T00:14:09,340 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-16T00:14:09,340 DEBUG [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, c7b798bc69dfc2fada33fccf8ca28883=TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883., 80495c60a5e712e96c604963e7816d5b=hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.}
2024-12-16T00:14:09,340 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c7b798bc69dfc2fada33fccf8ca28883 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-16T00:14:09,340 WARN  [WAL-Shutdown-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:09,340 DEBUG [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 80495c60a5e712e96c604963e7816d5b, c7b798bc69dfc2fada33fccf8ca28883
2024-12-16T00:14:09,340 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:14:09,340 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:14:09,361 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:14:09,362 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:14:09,362 ERROR [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(1664): Shutdown / close of WAL failed: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
2024-12-16T00:14:09,362 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:14:09,362 DEBUG [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(1665): Shutdown / close exception details:
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:09,362 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.87 KB heapSize=5.40 KB
2024-12-16T00:14:09,362 DEBUG [RS:1;b878af803003:36405 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:14:09,362 INFO  [RS:1;b878af803003:36405 {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:14:09,362 WARN  [RS_OPEN_META-regionserver/b878af803003:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:09,362 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C39441%2C1734308000099.meta:.meta(num 1734308001078) roll requested
2024-12-16T00:14:09,363 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:14:09,363 INFO  [RS:1;b878af803003:36405 {}] hbase.ChoreService(370): Chore service for: regionserver/b878af803003:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-16T00:14:09,363 INFO  [RS:1;b878af803003:36405 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-16T00:14:09,363 INFO  [RS:1;b878af803003:36405 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-16T00:14:09,363 INFO  [RS:1;b878af803003:36405 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-16T00:14:09,363 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C39441%2C1734308000099.meta.1734308049363.meta
2024-12-16T00:14:09,363 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:14:09,363 INFO  [RS:1;b878af803003:36405 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36405
2024-12-16T00:14:09,363 ERROR [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2808): ***** ABORTING region server b878af803003,39441,1734308000099: Unrecoverable exception while closing hbase:meta,,1.1588230740 *****
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:09,364 ERROR [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2815): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint]
2024-12-16T00:14:09,366 WARN  [Thread-815 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:09,366 WARN  [Thread-815 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741868_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK], DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]) is bad.
2024-12-16T00:14:09,366 WARN  [Thread-815 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741868_1052
2024-12-16T00:14:09,366 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for java.lang:type=Memory
2024-12-16T00:14:09,367 WARN  [Thread-815 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]
2024-12-16T00:14:09,367 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/.tmp/info/2d1d1f8547484516ad8988c5fa715f86 is 1080, key is row0014/info:/1734308049266/Put/seqid=0
2024-12-16T00:14:09,368 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC
2024-12-16T00:14:09,368 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication
2024-12-16T00:14:09,368 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server
2024-12-16T00:14:09,368 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2819): Dump of metrics as JSON on abort: {
  "beans": [
    {
      "name": "java.lang:type=Memory",
      "modelerType": "sun.management.MemoryImpl",
      "ObjectPendingFinalizationCount": 0,
      "HeapMemoryUsage": {
        "committed": 1048576000,
        "init": 1048576000,
        "max": 2306867200,
        "used": 234188752
      },
      "NonHeapMemoryUsage": {
        "committed": 161939456,
        "init": 7667712,
        "max": -1,
        "used": 160074200
      },
      "Verbose": false,
      "ObjectName": "java.lang:type=Memory"
    }
  ],
  "beans": [],
  "beans": [],
  "beans": []
}
2024-12-16T00:14:09,369 WARN  [Thread-815 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1053
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33583
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:09,369 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:48090 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741869_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10]'}, localName='127.0.0.1:44051', datanodeUuid='e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce', xmitsInProgress=0}:Exception transferring block BP-1525461463-172.17.0.2-1734307998365:blk_1073741869_1053 to mirror 127.0.0.1:33583
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:09,369 WARN  [Thread-815 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741869_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK], DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]) is bad.
2024-12-16T00:14:09,369 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:48090 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741869_1053] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-16T00:14:09,369 WARN  [Thread-815 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741869_1053
2024-12-16T00:14:09,369 WARN  [Thread-816 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:09,369 WARN  [Thread-816 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1525461463-172.17.0.2-1734307998365:blk_1073741870_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK], DatanodeInfoWithStorage[127.0.0.1:46225,DS-ae04eca9-a881-4228-b6b9-85ae9948933b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]) is bad.
2024-12-16T00:14:09,369 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1019009307_22 at /127.0.0.1:48090 [Receiving block BP-1525461463-172.17.0.2-1734307998365:blk_1073741869_1053] {}] datanode.DataXceiver(331): 127.0.0.1:44051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48090 dst: /127.0.0.1:44051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:09,370 WARN  [Thread-816 {}] hdfs.DataStreamer(1850): Abandoning BP-1525461463-172.17.0.2-1734307998365:blk_1073741870_1054
2024-12-16T00:14:09,370 WARN  [Thread-815 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]
2024-12-16T00:14:09,370 WARN  [Thread-816 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33583,DS-075199ab-2554-4f59-837f-37282af91993,DISK]
2024-12-16T00:14:09,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36405-0x1002c2f0d4a0003, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b878af803003,36405,1734308001768
2024-12-16T00:14:09,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:14:09,372 WARN  [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39029 {}] master.MasterRpcServices(626): b878af803003,39441,1734308000099 reported a fatal error:
***** ABORTING region server b878af803003,39441,1734308000099: Unrecoverable exception while closing hbase:meta,,1.1588230740 *****
Cause:
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191)
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064)
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967)
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168)
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721)
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644)
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627)
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408)
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707)

2024-12-16T00:14:09,379 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b878af803003,36405,1734308001768]
2024-12-16T00:14:09,379 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing b878af803003,36405,1734308001768; numProcessing=1
2024-12-16T00:14:09,382 WARN  [regionserver/b878af803003:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
2024-12-16T00:14:09,382 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta with entries=11, filesize=3.63 KB; new WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308049363.meta
2024-12-16T00:14:09,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41131 is added to blk_1073741872_1056 (size=14663)
2024-12-16T00:14:09,383 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34267:34267),(127.0.0.1/127.0.0.1:33049:33049)]
2024-12-16T00:14:09,383 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta is not closed yet, will try archiving it next time
2024-12-16T00:14:09,383 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:09,383 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43815,DS-f0103ffb-8953-400c-8557-7ca5936be765,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:09,383 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
2024-12-16T00:14:09,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741872_1056 (size=14663)
2024-12-16T00:14:09,383 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/.tmp/info/2d1d1f8547484516ad8988c5fa715f86
2024-12-16T00:14:09,384 WARN  [IPC Server handler 4 on default port 37443 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta has not been closed. Lease recovery is in progress. RecoveryId = 1057 for block blk_1073741834_1010
2024-12-16T00:14:09,384 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta after 1ms
2024-12-16T00:14:09,387 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/b878af803003,36405,1734308001768 already deleted, retry=false
2024-12-16T00:14:09,387 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; b878af803003,36405,1734308001768 expired; onlineServers=1
2024-12-16T00:14:09,389 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/.tmp/info/2d1d1f8547484516ad8988c5fa715f86 as hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/info/2d1d1f8547484516ad8988c5fa715f86
2024-12-16T00:14:09,395 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/info/2d1d1f8547484516ad8988c5fa715f86, entries=9, sequenceid=37, filesize=14.3 K
2024-12-16T00:14:09,396 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=0 B/0 for c7b798bc69dfc2fada33fccf8ca28883 in 56ms, sequenceid=37, compaction requested=true
2024-12-16T00:14:09,401 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c7b798bc69dfc2fada33fccf8ca28883/recovered.edits/40.seqid, newMaxSeqId=40, maxSeqId=1
2024-12-16T00:14:09,402 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:14:09,402 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c7b798bc69dfc2fada33fccf8ca28883:

2024-12-16T00:14:09,402 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1734308001920.c7b798bc69dfc2fada33fccf8ca28883.
2024-12-16T00:14:09,402 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 80495c60a5e712e96c604963e7816d5b, disabling compactions & flushes
2024-12-16T00:14:09,402 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:14:09,402 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:14:09,402 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b. after waiting 0 ms
2024-12-16T00:14:09,402 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:14:09,402 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 80495c60a5e712e96c604963e7816d5b:

2024-12-16T00:14:09,402 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:14:09,479 INFO  [RS:1;b878af803003:36405 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,36405,1734308001768; zookeeper connection closed.
2024-12-16T00:14:09,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36405-0x1002c2f0d4a0003, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:14:09,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36405-0x1002c2f0d4a0003, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:14:09,480 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28460e1b {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28460e1b
2024-12-16T00:14:09,498 INFO  [regionserver/b878af803003:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-16T00:14:09,498 INFO  [regionserver/b878af803003:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-16T00:14:09,540 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-16T00:14:09,540 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(3579): Received CLOSE for 80495c60a5e712e96c604963e7816d5b
2024-12-16T00:14:09,540 DEBUG [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 80495c60a5e712e96c604963e7816d5b
2024-12-16T00:14:09,540 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:14:09,540 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 80495c60a5e712e96c604963e7816d5b, disabling compactions & flushes
2024-12-16T00:14:09,541 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:14:09,541 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:14:09,541 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:14:09,541 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:14:09,541 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:14:09,541 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b. after waiting 0 ms
2024-12-16T00:14:09,541 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:14:09,541 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:14:09,541 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:14:09,541 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 80495c60a5e712e96c604963e7816d5b:

2024-12-16T00:14:09,541 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:meta,,1.1588230740
2024-12-16T00:14:09,541 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1734308001181.80495c60a5e712e96c604963e7816d5b.
2024-12-16T00:14:09,665 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308025549 to hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/oldWALs/b878af803003%2C39441%2C1734308000099.1734308025549
2024-12-16T00:14:09,667 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308029586 to hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/oldWALs/b878af803003%2C39441%2C1734308000099.1734308029586
2024-12-16T00:14:09,668 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.1734308029810 to hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/oldWALs/b878af803003%2C39441%2C1734308000099.1734308029810
2024-12-16T00:14:09,741 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1624): We were exiting though online regions are not empty, because some regions failed closing
2024-12-16T00:14:09,741 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1250): stopping server b878af803003,39441,1734308000099; all regions closed.
2024-12-16T00:14:09,741 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099
2024-12-16T00:14:09,744 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741871_1055 (size=93)
2024-12-16T00:14:09,745 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41131 is added to blk_1073741871_1055 (size=93)
2024-12-16T00:14:09,894 INFO  [regionserver/b878af803003:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:14:10,288 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@a1ff948[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44051, datanodeUuid=e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce, infoPort=33049, infoSecurePort=0, ipcPort=33223, storageInfo=lv=-57;cid=testClusterID;nsid=776977188;c=1734307998365):Failed to transfer BP-1525461463-172.17.0.2-1734307998365:blk_1073741858_1041 to 127.0.0.1:33583 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:10,496 INFO  [regionserver/b878af803003:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:14:10,736 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@5cc315f1 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1525461463-172.17.0.2-1734307998365:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:43815,null,null])
java.net.ConnectException: Call From b878af803003/172.17.0.2 to localhost:38427 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-16T00:14:13,385 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta after 4002ms
2024-12-16T00:14:13,703 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741830_1047 (size=47160)
2024-12-16T00:14:14,405 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,417 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,420 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,420 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,425 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,425 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,746 ERROR [WAL-Shutdown-0 {}] wal.FSHLog(508): We have waited 5 seconds but the close of writer(s) doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-16T00:14:14,747 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099
2024-12-16T00:14:14,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741864_1048 (size=13514)
2024-12-16T00:14:14,754 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41131 is added to blk_1073741864_1048 (size=13514)
2024-12-16T00:14:14,755 DEBUG [RS:0;b878af803003:39441 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:14:14,756 INFO  [RS:0;b878af803003:39441 {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:14:14,756 INFO  [RS:0;b878af803003:39441 {}] hbase.ChoreService(370): Chore service for: regionserver/b878af803003:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-16T00:14:14,756 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:14:14,757 INFO  [RS:0;b878af803003:39441 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:39441
2024-12-16T00:14:14,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b878af803003,39441,1734308000099
2024-12-16T00:14:14,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:14:14,888 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b878af803003,39441,1734308000099]
2024-12-16T00:14:14,888 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing b878af803003,39441,1734308000099; numProcessing=2
2024-12-16T00:14:14,896 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/b878af803003,39441,1734308000099 already deleted, retry=false
2024-12-16T00:14:14,896 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; b878af803003,39441,1734308000099 expired; onlineServers=0
2024-12-16T00:14:14,897 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,39029,1734307999962' *****
2024-12-16T00:14:14,897 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-16T00:14:14,897 DEBUG [M:0;b878af803003:39029 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e2b983, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:14:14,898 INFO  [M:0;b878af803003:39029 {}] regionserver.HRegionServer(1224): stopping server b878af803003,39029,1734307999962
2024-12-16T00:14:14,898 INFO  [M:0;b878af803003:39029 {}] regionserver.HRegionServer(1250): stopping server b878af803003,39029,1734307999962; all regions closed.
2024-12-16T00:14:14,898 DEBUG [M:0;b878af803003:39029 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:14:14,898 DEBUG [M:0;b878af803003:39029 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-16T00:14:14,899 DEBUG [M:0;b878af803003:39029 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-16T00:14:14,899 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308000415 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308000415,5,FailOnTimeoutGroup]
2024-12-16T00:14:14,899 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-16T00:14:14,899 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308000414 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308000414,5,FailOnTimeoutGroup]
2024-12-16T00:14:14,900 INFO  [M:0;b878af803003:39029 {}] hbase.ChoreService(370): Chore service for: master/b878af803003:0 had [] on shutdown
2024-12-16T00:14:14,900 DEBUG [M:0;b878af803003:39029 {}] master.HMaster(1733): Stopping service threads
2024-12-16T00:14:14,900 INFO  [M:0;b878af803003:39029 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-16T00:14:14,900 INFO  [M:0;b878af803003:39029 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-16T00:14:14,901 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-16T00:14:14,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-16T00:14:14,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:14,905 DEBUG [M:0;b878af803003:39029 {}] zookeeper.ZKUtil(347): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-16T00:14:14,905 WARN  [M:0;b878af803003:39029 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-16T00:14:14,905 INFO  [M:0;b878af803003:39029 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-16T00:14:14,905 INFO  [M:0;b878af803003:39029 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-16T00:14:14,905 DEBUG [M:0;b878af803003:39029 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:14:14,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:14:14,905 INFO  [M:0;b878af803003:39029 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:14:14,905 DEBUG [M:0;b878af803003:39029 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:14:14,905 DEBUG [M:0;b878af803003:39029 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:14:14,906 DEBUG [M:0;b878af803003:39029 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:14:14,906 INFO  [M:0;b878af803003:39029 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.09 KB heapSize=49.30 KB
2024-12-16T00:14:14,908 WARN  [sync.3 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]
2024-12-16T00:14:14,908 WARN  [sync.3 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44051,DS-64c58a24-5d97-4ae2-b99c-46053965fe45,DISK]]
2024-12-16T00:14:14,908 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C39029%2C1734307999962:(num 1734308030414) roll requested
2024-12-16T00:14:14,909 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C39029%2C1734307999962.1734308054908
2024-12-16T00:14:14,916 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308030414 with entries=1, filesize=349 B; new WAL /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308054908
2024-12-16T00:14:14,916 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34267:34267),(127.0.0.1/127.0.0.1:33049:33049)]
2024-12-16T00:14:14,916 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308030414 is not closed yet, will try archiving it next time
2024-12-16T00:14:14,917 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741863_1046 (size=357)
2024-12-16T00:14:14,925 DEBUG [M:0;b878af803003:39029 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/034b2383f4cf4676806e76b6abca6cba is 82, key is hbase:meta,,1/info:regioninfo/1734308001105/Put/seqid=0
2024-12-16T00:14:14,927 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-16T00:14:14,928 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41131 is added to blk_1073741874_1059 (size=5672)
2024-12-16T00:14:14,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741874_1059 (size=5672)
2024-12-16T00:14:14,930 INFO  [M:0;b878af803003:39029 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/034b2383f4cf4676806e76b6abca6cba
2024-12-16T00:14:14,944 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,945 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,949 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,949 DEBUG [M:0;b878af803003:39029 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/67d23cd56fc2436ca21b768904d0ab71 is 775, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1734308002332/Put/seqid=0
2024-12-16T00:14:14,949 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,950 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:14,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41131 is added to blk_1073741875_1060 (size=7466)
2024-12-16T00:14:14,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741875_1060 (size=7466)
2024-12-16T00:14:14,955 INFO  [M:0;b878af803003:39029 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.42 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/67d23cd56fc2436ca21b768904d0ab71
2024-12-16T00:14:14,971 DEBUG [M:0;b878af803003:39029 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d8bbad6e87294eb38d680c5054ffaef1 is 69, key is b878af803003,36405,1734308001768/rs:state/1734308001818/Put/seqid=0
2024-12-16T00:14:14,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41131 is added to blk_1073741876_1061 (size=5224)
2024-12-16T00:14:14,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741876_1061 (size=5224)
2024-12-16T00:14:14,977 INFO  [M:0;b878af803003:39029 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d8bbad6e87294eb38d680c5054ffaef1
2024-12-16T00:14:14,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:14:14,988 INFO  [RS:0;b878af803003:39441 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,39441,1734308000099; zookeeper connection closed.
2024-12-16T00:14:14,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39441-0x1002c2f0d4a0001, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:14:14,989 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@532cfacc {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@532cfacc
2024-12-16T00:14:14,989 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-16T00:14:14,998 DEBUG [M:0;b878af803003:39029 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/74421213273640f798552b601e119e81 is 52, key is load_balancer_on/state:d/1734308001755/Put/seqid=0
2024-12-16T00:14:15,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741877_1062 (size=5056)
2024-12-16T00:14:15,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41131 is added to blk_1073741877_1062 (size=5056)
2024-12-16T00:14:15,003 INFO  [M:0;b878af803003:39029 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/74421213273640f798552b601e119e81
2024-12-16T00:14:15,009 DEBUG [M:0;b878af803003:39029 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/034b2383f4cf4676806e76b6abca6cba as hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/034b2383f4cf4676806e76b6abca6cba
2024-12-16T00:14:15,015 INFO  [M:0;b878af803003:39029 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/034b2383f4cf4676806e76b6abca6cba, entries=8, sequenceid=97, filesize=5.5 K
2024-12-16T00:14:15,016 DEBUG [M:0;b878af803003:39029 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/67d23cd56fc2436ca21b768904d0ab71 as hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/67d23cd56fc2436ca21b768904d0ab71
2024-12-16T00:14:15,022 INFO  [M:0;b878af803003:39029 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/67d23cd56fc2436ca21b768904d0ab71, entries=11, sequenceid=97, filesize=7.3 K
2024-12-16T00:14:15,023 DEBUG [M:0;b878af803003:39029 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d8bbad6e87294eb38d680c5054ffaef1 as hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d8bbad6e87294eb38d680c5054ffaef1
2024-12-16T00:14:15,028 INFO  [M:0;b878af803003:39029 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d8bbad6e87294eb38d680c5054ffaef1, entries=2, sequenceid=97, filesize=5.1 K
2024-12-16T00:14:15,029 DEBUG [M:0;b878af803003:39029 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/74421213273640f798552b601e119e81 as hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/74421213273640f798552b601e119e81
2024-12-16T00:14:15,034 INFO  [M:0;b878af803003:39029 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/74421213273640f798552b601e119e81, entries=1, sequenceid=97, filesize=4.9 K
2024-12-16T00:14:15,035 INFO  [M:0;b878af803003:39029 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.09 KB/41051, heapSize ~49.23 KB/50416, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 129ms, sequenceid=97, compaction requested=false
2024-12-16T00:14:15,036 INFO  [M:0;b878af803003:39029 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:14:15,036 DEBUG [M:0;b878af803003:39029 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:14:15,036 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962
2024-12-16T00:14:15,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44051 is added to blk_1073741873_1058 (size=493)
2024-12-16T00:14:15,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41131 is added to blk_1073741873_1058 (size=493)
2024-12-16T00:14:15,320 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308000230 to hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/oldWALs/b878af803003%2C39029%2C1734307999962.1734308000230
2024-12-16T00:14:15,326 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/oldWALs/b878af803003%2C39029%2C1734307999962.1734308000230 to hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/oldWALs/b878af803003%2C39029%2C1734307999962.1734308000230$masterlocalwal$
2024-12-16T00:14:15,326 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/WALs/b878af803003,39029,1734307999962/b878af803003%2C39029%2C1734307999962.1734308030414 to hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/oldWALs/b878af803003%2C39029%2C1734307999962.1734308030414
2024-12-16T00:14:15,331 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/MasterData/oldWALs/b878af803003%2C39029%2C1734307999962.1734308030414 to hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/oldWALs/b878af803003%2C39029%2C1734307999962.1734308030414$masterlocalwal$
2024-12-16T00:14:15,331 INFO  [M:0;b878af803003:39029 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-16T00:14:15,331 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:14:15,331 INFO  [M:0;b878af803003:39029 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:39029
2024-12-16T00:14:15,363 DEBUG [M:0;b878af803003:39029 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/b878af803003,39029,1734307999962 already deleted, retry=false
2024-12-16T00:14:15,387 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:15,472 INFO  [M:0;b878af803003:39029 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,39029,1734307999962; zookeeper connection closed.
2024-12-16T00:14:15,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:14:15,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39029-0x1002c2f0d4a0000, quorum=127.0.0.1:50614, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:14:15,479 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7d2e9057{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:15,481 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3ce3e309{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:14:15,481 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:14:15,481 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d5fd30b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:14:15,482 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78252885{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,STOPPED}
2024-12-16T00:14:15,484 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:14:15,484 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:14:15,483 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@78c404ca {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1525461463-172.17.0.2-1734307998365:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:43815,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:38427 , LocalHost:localPort b878af803003/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-16T00:14:15,484 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1525461463-172.17.0.2-1734307998365 (Datanode Uuid ed51c821-41de-47d1-8ef3-1a57dee79fe3) service to localhost/127.0.0.1:37443
2024-12-16T00:14:15,484 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:14:15,484 ERROR [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@78c404ca {}] datanode.DataNode(1743): Cannot find BPOfferService for reporting block received for bpid=BP-1525461463-172.17.0.2-1734307998365
2024-12-16T00:14:15,485 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data3/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:15,485 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data4/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:15,485 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:14:15,487 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@280164fa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:15,487 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1e996930{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:14:15,487 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:14:15,488 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58d3f3f2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:14:15,488 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fb180fe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,STOPPED}
2024-12-16T00:14:15,489 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:14:15,489 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:14:15,489 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:14:15,489 WARN  [BP-1525461463-172.17.0.2-1734307998365 heartbeating to localhost/127.0.0.1:37443 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1525461463-172.17.0.2-1734307998365 (Datanode Uuid e66bf2a2-3d4c-4d6e-aede-0f85c2cf70ce) service to localhost/127.0.0.1:37443
2024-12-16T00:14:15,490 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data9/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:15,490 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/cluster_83ee28ff-b5a2-8f15-c2fa-28787360e8d5/dfs/data/data10/current/BP-1525461463-172.17.0.2-1734307998365 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:15,490 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:14:15,495 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6f234dd7{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:14:15,496 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3afa3b9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:14:15,496 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:14:15,496 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25d29461{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:14:15,496 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c4a27fe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir/,STOPPED}
2024-12-16T00:14:15,503 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-16T00:14:15,533 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-16T00:14:15,539 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=86 (was 65)
Potentially hanging thread: nioEventLoopGroup-14-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$$Lambda$797/0x00007faf04b67210.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:37443
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (679588824) connection to localhost/127.0.0.1:37443 from jenkins.hfs.1
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37443
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37443
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:37443
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (679588824) connection to localhost/127.0.0.1:37443 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (679588824) connection to localhost/127.0.0.1:37443 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Abort regionserver monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-15-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:37443
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37443
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=424 (was 408) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=33 (was 49), ProcessCount=11 (was 11), AvailableMemoryMB=12134 (was 12491)
2024-12-16T00:14:15,545 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=86, OpenFileDescriptor=424, MaxFileDescriptor=1048576, SystemLoadAverage=33, ProcessCount=11, AvailableMemoryMB=12134
2024-12-16T00:14:15,545 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-16T00:14:15,545 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.log.dir so I do NOT create it in target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f
2024-12-16T00:14:15,545 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b1e2482a-eb9c-e32e-7a8f-8a74f75c4ea9/hadoop.tmp.dir so I do NOT create it in target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f
2024-12-16T00:14:15,545 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687, deleteOnExit=true
2024-12-16T00:14:15,545 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-16T00:14:15,545 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/test.cache.data in system properties and HBase conf
2024-12-16T00:14:15,545 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.tmp.dir in system properties and HBase conf
2024-12-16T00:14:15,545 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir in system properties and HBase conf
2024-12-16T00:14:15,545 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-16T00:14:15,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-16T00:14:15,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-16T00:14:15,546 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-16T00:14:15,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:14:15,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:14:15,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-16T00:14:15,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:14:15,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-16T00:14:15,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-16T00:14:15,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:14:15,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:14:15,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-16T00:14:15,547 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/nfs.dump.dir in system properties and HBase conf
2024-12-16T00:14:15,547 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/java.io.tmpdir in system properties and HBase conf
2024-12-16T00:14:15,547 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:14:15,547 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-16T00:14:15,547 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-16T00:14:15,557 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:14:15,878 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:14:15,882 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:14:15,883 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:14:15,883 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:14:15,883 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:14:15,884 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:14:15,884 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@61b03158{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:14:15,885 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@339df197{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:14:15,974 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@56bd30de{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/java.io.tmpdir/jetty-localhost-35683-hadoop-hdfs-3_4_1-tests_jar-_-any-2232094063271391574/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:14:15,974 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@377bc93b{HTTP/1.1, (http/1.1)}{localhost:35683}
2024-12-16T00:14:15,974 INFO  [Time-limited test {}] server.Server(415): Started @174639ms
2024-12-16T00:14:15,985 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:14:16,136 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:14:16,138 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:14:16,139 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:14:16,139 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:14:16,139 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:14:16,139 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b1885ce{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:14:16,140 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5dbd1f1f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:14:16,228 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@778a07eb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/java.io.tmpdir/jetty-localhost-42997-hadoop-hdfs-3_4_1-tests_jar-_-any-10180242964625420004/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:16,229 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@578de26e{HTTP/1.1, (http/1.1)}{localhost:42997}
2024-12-16T00:14:16,229 INFO  [Time-limited test {}] server.Server(415): Started @174893ms
2024-12-16T00:14:16,230 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:14:16,253 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:14:16,255 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:14:16,257 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:14:16,257 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:14:16,257 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:14:16,259 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8f79290{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:14:16,259 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ffe559e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:14:16,348 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1f1c6e43{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/java.io.tmpdir/jetty-localhost-44837-hadoop-hdfs-3_4_1-tests_jar-_-any-7119634644990207763/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:16,348 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b10eec4{HTTP/1.1, (http/1.1)}{localhost:44837}
2024-12-16T00:14:16,348 INFO  [Time-limited test {}] server.Server(415): Started @175013ms
2024-12-16T00:14:16,349 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:14:16,388 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:16,862 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-16T00:14:16,862 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-16T00:14:16,989 WARN  [Thread-955 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data1/current/BP-1291920219-172.17.0.2-1734308055567/current, will proceed with Du for space computation calculation, 
2024-12-16T00:14:16,989 WARN  [Thread-956 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data2/current/BP-1291920219-172.17.0.2-1734308055567/current, will proceed with Du for space computation calculation, 
2024-12-16T00:14:17,007 WARN  [Thread-919 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:14:17,009 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xac9d46ca8fcadbf8 with lease ID 0x36234a392fa404c4: Processing first storage report for DS-e5b6417b-1e48-469c-9939-0a3674fe0f94 from datanode DatanodeRegistration(127.0.0.1:36981, datanodeUuid=d9c191f8-4d5d-414a-a8eb-0f32d2637f45, infoPort=36231, infoSecurePort=0, ipcPort=39423, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567)
2024-12-16T00:14:17,009 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xac9d46ca8fcadbf8 with lease ID 0x36234a392fa404c4: from storage DS-e5b6417b-1e48-469c-9939-0a3674fe0f94 node DatanodeRegistration(127.0.0.1:36981, datanodeUuid=d9c191f8-4d5d-414a-a8eb-0f32d2637f45, infoPort=36231, infoSecurePort=0, ipcPort=39423, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:14:17,010 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xac9d46ca8fcadbf8 with lease ID 0x36234a392fa404c4: Processing first storage report for DS-245570bb-64dc-4a1e-914c-ddc22a18bb81 from datanode DatanodeRegistration(127.0.0.1:36981, datanodeUuid=d9c191f8-4d5d-414a-a8eb-0f32d2637f45, infoPort=36231, infoSecurePort=0, ipcPort=39423, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567)
2024-12-16T00:14:17,010 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xac9d46ca8fcadbf8 with lease ID 0x36234a392fa404c4: from storage DS-245570bb-64dc-4a1e-914c-ddc22a18bb81 node DatanodeRegistration(127.0.0.1:36981, datanodeUuid=d9c191f8-4d5d-414a-a8eb-0f32d2637f45, infoPort=36231, infoSecurePort=0, ipcPort=39423, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:14:17,089 WARN  [Thread-966 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data3/current/BP-1291920219-172.17.0.2-1734308055567/current, will proceed with Du for space computation calculation, 
2024-12-16T00:14:17,089 WARN  [Thread-967 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data4/current/BP-1291920219-172.17.0.2-1734308055567/current, will proceed with Du for space computation calculation, 
2024-12-16T00:14:17,105 WARN  [Thread-942 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:14:17,107 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3af5a81456488cce with lease ID 0x36234a392fa404c5: Processing first storage report for DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1 from datanode DatanodeRegistration(127.0.0.1:43245, datanodeUuid=9e9f74ac-5ae4-4b7e-89af-14153e85bce4, infoPort=45575, infoSecurePort=0, ipcPort=45595, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567)
2024-12-16T00:14:17,107 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3af5a81456488cce with lease ID 0x36234a392fa404c5: from storage DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1 node DatanodeRegistration(127.0.0.1:43245, datanodeUuid=9e9f74ac-5ae4-4b7e-89af-14153e85bce4, infoPort=45575, infoSecurePort=0, ipcPort=45595, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:14:17,107 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3af5a81456488cce with lease ID 0x36234a392fa404c5: Processing first storage report for DS-4b913d61-8138-4068-9ea5-08bb0f1bc574 from datanode DatanodeRegistration(127.0.0.1:43245, datanodeUuid=9e9f74ac-5ae4-4b7e-89af-14153e85bce4, infoPort=45575, infoSecurePort=0, ipcPort=45595, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567)
2024-12-16T00:14:17,107 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3af5a81456488cce with lease ID 0x36234a392fa404c5: from storage DS-4b913d61-8138-4068-9ea5-08bb0f1bc574 node DatanodeRegistration(127.0.0.1:43245, datanodeUuid=9e9f74ac-5ae4-4b7e-89af-14153e85bce4, infoPort=45575, infoSecurePort=0, ipcPort=45595, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:14:17,181 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f
2024-12-16T00:14:17,203 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/zookeeper_0, clientPort=64341, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-16T00:14:17,205 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=64341
2024-12-16T00:14:17,205 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:14:17,207 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:14:17,218 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43245 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:14:17,218 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36981 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:14:17,219 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc with version=8
2024-12-16T00:14:17,219 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/hbase-staging
2024-12-16T00:14:17,221 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/b878af803003:0 server-side Connection retries=45
2024-12-16T00:14:17,221 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:14:17,221 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:14:17,221 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:14:17,221 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:14:17,221 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:14:17,221 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:14:17,222 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:14:17,222 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45255
2024-12-16T00:14:17,223 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:14:17,224 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:14:17,226 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:45255 connecting to ZooKeeper ensemble=127.0.0.1:64341
2024-12-16T00:14:17,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:452550x0, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:14:17,284 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45255-0x1002c2fecf20000 connected
2024-12-16T00:14:17,363 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:14:17,363 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:14:17,364 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:14:17,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45255
2024-12-16T00:14:17,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45255
2024-12-16T00:14:17,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45255
2024-12-16T00:14:17,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45255
2024-12-16T00:14:17,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45255
2024-12-16T00:14:17,365 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc, hbase.cluster.distributed=false
2024-12-16T00:14:17,378 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/b878af803003:0 server-side Connection retries=45
2024-12-16T00:14:17,378 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:14:17,378 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:14:17,378 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:14:17,378 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:14:17,378 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:14:17,378 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:14:17,379 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:14:17,379 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:41029
2024-12-16T00:14:17,379 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-16T00:14:17,380 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-16T00:14:17,380 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:14:17,382 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:14:17,384 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:41029 connecting to ZooKeeper ensemble=127.0.0.1:64341
2024-12-16T00:14:17,389 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:17,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:410290x0, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:14:17,395 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:410290x0, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:14:17,395 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41029-0x1002c2fecf20001 connected
2024-12-16T00:14:17,396 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:14:17,396 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:14:17,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41029
2024-12-16T00:14:17,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41029
2024-12-16T00:14:17,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41029
2024-12-16T00:14:17,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41029
2024-12-16T00:14:17,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41029
2024-12-16T00:14:17,400 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/b878af803003,45255,1734308057221
2024-12-16T00:14:17,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:14:17,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:14:17,411 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b878af803003,45255,1734308057221
2024-12-16T00:14:17,411 DEBUG [M:0;b878af803003:45255 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b878af803003:45255
2024-12-16T00:14:17,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:14:17,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:14:17,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:17,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:17,421 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:14:17,422 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b878af803003,45255,1734308057221 from backup master directory
2024-12-16T00:14:17,422 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:14:17,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b878af803003,45255,1734308057221
2024-12-16T00:14:17,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:14:17,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:14:17,429 WARN  [master/b878af803003:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:14:17,429 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b878af803003,45255,1734308057221
2024-12-16T00:14:17,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43245 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:14:17,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36981 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:14:17,443 DEBUG [master/b878af803003:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/hbase.id with ID: 004ec0c6-d005-4521-b2ca-b23f9d5ecab4
2024-12-16T00:14:17,454 INFO  [master/b878af803003:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:14:17,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:17,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:17,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43245 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:14:17,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36981 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:14:17,470 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-16T00:14:17,471 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-16T00:14:17,471 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:14:17,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36981 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:14:17,481 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43245 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:14:17,482 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store
2024-12-16T00:14:17,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43245 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:14:17,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36981 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:14:17,489 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:14:17,489 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:14:17,489 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:14:17,489 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:14:17,489 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:14:17,489 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:14:17,489 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:14:17,489 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:14:17,490 WARN  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/.initializing
2024-12-16T00:14:17,490 DEBUG [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221
2024-12-16T00:14:17,493 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C45255%2C1734308057221, suffix=, logDir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221, archiveDir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/oldWALs, maxLogs=10
2024-12-16T00:14:17,493 INFO  [master/b878af803003:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45255%2C1734308057221.1734308057493
2024-12-16T00:14:17,499 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221/b878af803003%2C45255%2C1734308057221.1734308057493
2024-12-16T00:14:17,499 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45575:45575),(127.0.0.1/127.0.0.1:36231:36231)]
2024-12-16T00:14:17,499 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:14:17,499 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:14:17,499 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:14:17,500 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:14:17,501 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:14:17,503 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-16T00:14:17,503 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:17,503 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:14:17,503 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:14:17,505 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-16T00:14:17,505 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:17,505 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:14:17,505 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:14:17,506 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-16T00:14:17,506 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:17,507 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:14:17,507 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:14:17,508 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-16T00:14:17,508 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:17,509 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:14:17,509 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:14:17,510 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:14:17,512 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-16T00:14:17,513 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:14:17,515 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:14:17,515 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=778139, jitterRate=-0.010546058416366577}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-16T00:14:17,516 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:14:17,516 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-16T00:14:17,520 DEBUG [master/b878af803003:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f6ca39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:14:17,520 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-16T00:14:17,521 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-16T00:14:17,521 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-16T00:14:17,521 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-16T00:14:17,521 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-16T00:14:17,522 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-16T00:14:17,522 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-16T00:14:17,525 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-16T00:14:17,525 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-16T00:14:17,536 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-16T00:14:17,536 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-16T00:14:17,537 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-16T00:14:17,546 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-16T00:14:17,546 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-16T00:14:17,547 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-16T00:14:17,554 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-16T00:14:17,555 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-16T00:14:17,562 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-16T00:14:17,564 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-16T00:14:17,571 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-16T00:14:17,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:14:17,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:14:17,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:17,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:17,580 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=b878af803003,45255,1734308057221, sessionid=0x1002c2fecf20000, setting cluster-up flag (Was=false)
2024-12-16T00:14:17,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:17,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:17,621 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-16T00:14:17,622 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,45255,1734308057221
2024-12-16T00:14:17,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:17,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:17,662 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-16T00:14:17,663 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,45255,1734308057221
2024-12-16T00:14:17,666 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-16T00:14:17,667 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-16T00:14:17,667 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-16T00:14:17,667 DEBUG [master/b878af803003:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b878af803003,45255,1734308057221
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-16T00:14:17,667 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:14:17,667 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:14:17,667 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:14:17,667 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:14:17,667 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b878af803003:0, corePoolSize=10, maxPoolSize=10
2024-12-16T00:14:17,668 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,668 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:14:17,668 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,669 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734308087669
2024-12-16T00:14:17,669 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-16T00:14:17,669 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-16T00:14:17,669 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-16T00:14:17,669 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-16T00:14:17,669 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-16T00:14:17,669 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-16T00:14:17,669 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:14:17,670 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:17,670 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-16T00:14:17,670 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-16T00:14:17,670 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-16T00:14:17,670 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-16T00:14:17,671 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-16T00:14:17,671 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-16T00:14:17,671 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:17,671 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308057671,5,FailOnTimeoutGroup]
2024-12-16T00:14:17,671 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:14:17,672 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308057671,5,FailOnTimeoutGroup]
2024-12-16T00:14:17,672 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:17,672 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-16T00:14:17,672 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:17,672 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:17,677 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43245 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:14:17,678 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36981 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:14:17,679 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-16T00:14:17,679 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc
2024-12-16T00:14:17,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43245 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:14:17,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36981 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:14:17,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:14:17,686 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:14:17,687 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:14:17,687 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:17,687 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:14:17,688 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:14:17,689 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:14:17,689 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:17,689 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:14:17,689 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:14:17,690 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:14:17,690 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:17,691 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:14:17,692 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740
2024-12-16T00:14:17,692 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740
2024-12-16T00:14:17,694 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:14:17,695 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:14:17,697 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:14:17,698 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=880535, jitterRate=0.11965902149677277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:14:17,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:14:17,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:14:17,698 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:14:17,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:14:17,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:14:17,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:14:17,699 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-16T00:14:17,699 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:14:17,700 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:14:17,700 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-16T00:14:17,700 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-16T00:14:17,701 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-16T00:14:17,702 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-16T00:14:17,710 DEBUG [RS:0;b878af803003:41029 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b878af803003:41029
2024-12-16T00:14:17,711 INFO  [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1008): ClusterId : 004ec0c6-d005-4521-b2ca-b23f9d5ecab4
2024-12-16T00:14:17,711 DEBUG [RS:0;b878af803003:41029 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-16T00:14:17,722 DEBUG [RS:0;b878af803003:41029 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-16T00:14:17,722 DEBUG [RS:0;b878af803003:41029 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-16T00:14:17,730 DEBUG [RS:0;b878af803003:41029 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-16T00:14:17,730 DEBUG [RS:0;b878af803003:41029 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc99a33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:14:17,731 DEBUG [RS:0;b878af803003:41029 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd4d0f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:14:17,731 INFO  [RS:0;b878af803003:41029 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-16T00:14:17,731 INFO  [RS:0;b878af803003:41029 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-16T00:14:17,731 DEBUG [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-16T00:14:17,732 INFO  [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(3073): reportForDuty to master=b878af803003,45255,1734308057221 with isa=b878af803003/172.17.0.2:41029, startcode=1734308057378
2024-12-16T00:14:17,732 DEBUG [RS:0;b878af803003:41029 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:14:17,734 INFO  [RS-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40901, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:14:17,735 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45255 {}] master.ServerManager(332): Checking decommissioned status of RegionServer b878af803003,41029,1734308057378
2024-12-16T00:14:17,735 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45255 {}] master.ServerManager(486): Registering regionserver=b878af803003,41029,1734308057378
2024-12-16T00:14:17,737 DEBUG [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc
2024-12-16T00:14:17,737 DEBUG [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:42703
2024-12-16T00:14:17,737 DEBUG [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-16T00:14:17,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:14:17,746 DEBUG [RS:0;b878af803003:41029 {}] zookeeper.ZKUtil(111): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b878af803003,41029,1734308057378
2024-12-16T00:14:17,747 WARN  [RS:0;b878af803003:41029 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:14:17,747 INFO  [RS:0;b878af803003:41029 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:14:17,747 DEBUG [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378
2024-12-16T00:14:17,747 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b878af803003,41029,1734308057378]
2024-12-16T00:14:17,750 DEBUG [RS:0;b878af803003:41029 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-16T00:14:17,750 INFO  [RS:0;b878af803003:41029 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-16T00:14:17,751 INFO  [RS:0;b878af803003:41029 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-16T00:14:17,751 INFO  [RS:0;b878af803003:41029 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-16T00:14:17,752 INFO  [RS:0;b878af803003:41029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:17,752 INFO  [RS:0;b878af803003:41029 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-16T00:14:17,753 INFO  [RS:0;b878af803003:41029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:14:17,753 DEBUG [RS:0;b878af803003:41029 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:14:17,754 INFO  [RS:0;b878af803003:41029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:17,754 INFO  [RS:0;b878af803003:41029 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:17,754 INFO  [RS:0;b878af803003:41029 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:17,754 INFO  [RS:0;b878af803003:41029 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:17,754 INFO  [RS:0;b878af803003:41029 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,41029,1734308057378-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:14:17,766 INFO  [RS:0;b878af803003:41029 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-16T00:14:17,767 INFO  [RS:0;b878af803003:41029 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,41029,1734308057378-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:17,778 INFO  [RS:0;b878af803003:41029 {}] regionserver.Replication(204): b878af803003,41029,1734308057378 started
2024-12-16T00:14:17,778 INFO  [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1767): Serving as b878af803003,41029,1734308057378, RpcServer on b878af803003/172.17.0.2:41029, sessionid=0x1002c2fecf20001
2024-12-16T00:14:17,779 DEBUG [RS:0;b878af803003:41029 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-16T00:14:17,779 DEBUG [RS:0;b878af803003:41029 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b878af803003,41029,1734308057378
2024-12-16T00:14:17,779 DEBUG [RS:0;b878af803003:41029 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,41029,1734308057378'
2024-12-16T00:14:17,779 DEBUG [RS:0;b878af803003:41029 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-16T00:14:17,779 DEBUG [RS:0;b878af803003:41029 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-16T00:14:17,780 DEBUG [RS:0;b878af803003:41029 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-16T00:14:17,780 DEBUG [RS:0;b878af803003:41029 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-16T00:14:17,780 DEBUG [RS:0;b878af803003:41029 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b878af803003,41029,1734308057378
2024-12-16T00:14:17,780 DEBUG [RS:0;b878af803003:41029 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,41029,1734308057378'
2024-12-16T00:14:17,780 DEBUG [RS:0;b878af803003:41029 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-16T00:14:17,780 DEBUG [RS:0;b878af803003:41029 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-16T00:14:17,781 DEBUG [RS:0;b878af803003:41029 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-16T00:14:17,781 INFO  [RS:0;b878af803003:41029 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-16T00:14:17,781 INFO  [RS:0;b878af803003:41029 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-16T00:14:17,853 WARN  [b878af803003:45255 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-16T00:14:17,884 INFO  [RS:0;b878af803003:41029 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C41029%2C1734308057378, suffix=, logDir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378, archiveDir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/oldWALs, maxLogs=32
2024-12-16T00:14:17,886 INFO  [RS:0;b878af803003:41029 {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C41029%2C1734308057378.1734308057886
2024-12-16T00:14:17,895 INFO  [RS:0;b878af803003:41029 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886
2024-12-16T00:14:17,895 DEBUG [RS:0;b878af803003:41029 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36231:36231),(127.0.0.1/127.0.0.1:45575:45575)]
2024-12-16T00:14:18,103 DEBUG [b878af803003:45255 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-16T00:14:18,104 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b878af803003,41029,1734308057378
2024-12-16T00:14:18,108 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,41029,1734308057378, state=OPENING
2024-12-16T00:14:18,146 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-16T00:14:18,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:18,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:18,156 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=b878af803003,41029,1734308057378}]
2024-12-16T00:14:18,156 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:14:18,156 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:14:18,313 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to b878af803003,41029,1734308057378
2024-12-16T00:14:18,314 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-16T00:14:18,319 INFO  [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37576, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-16T00:14:18,325 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-16T00:14:18,325 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:14:18,328 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C41029%2C1734308057378.meta, suffix=.meta, logDir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378, archiveDir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/oldWALs, maxLogs=32
2024-12-16T00:14:18,330 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C41029%2C1734308057378.meta.1734308058330.meta
2024-12-16T00:14:18,337 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.meta.1734308058330.meta
2024-12-16T00:14:18,337 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36231:36231),(127.0.0.1/127.0.0.1:45575:45575)]
2024-12-16T00:14:18,337 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:14:18,338 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-16T00:14:18,338 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-16T00:14:18,338 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-16T00:14:18,338 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-16T00:14:18,338 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:14:18,338 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-16T00:14:18,338 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-16T00:14:18,340 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:14:18,341 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:14:18,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:18,342 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:14:18,342 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:14:18,343 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:14:18,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:18,343 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:14:18,343 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:14:18,344 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:14:18,344 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:18,345 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:14:18,346 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740
2024-12-16T00:14:18,347 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740
2024-12-16T00:14:18,349 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:14:18,350 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:14:18,351 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=831336, jitterRate=0.05709889531135559}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:14:18,351 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:14:18,352 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734308058313
2024-12-16T00:14:18,354 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-16T00:14:18,354 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-16T00:14:18,355 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,41029,1734308057378
2024-12-16T00:14:18,356 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,41029,1734308057378, state=OPEN
2024-12-16T00:14:18,389 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:18,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:14:18,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:14:18,400 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:14:18,400 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:14:18,405 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-16T00:14:18,405 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=b878af803003,41029,1734308057378 in 244 msec
2024-12-16T00:14:18,410 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-16T00:14:18,410 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 705 msec
2024-12-16T00:14:18,414 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 747 msec
2024-12-16T00:14:18,414 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734308058414, completionTime=-1
2024-12-16T00:14:18,414 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-16T00:14:18,415 DEBUG [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-16T00:14:18,415 DEBUG [hconnection-0x243b9d2a-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:14:18,417 INFO  [RS-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37588, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:14:18,418 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-16T00:14:18,418 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734308118418
2024-12-16T00:14:18,418 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734308178418
2024-12-16T00:14:18,418 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec
2024-12-16T00:14:18,480 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,45255,1734308057221-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:18,480 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,45255,1734308057221-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:18,480 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,45255,1734308057221-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:18,480 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b878af803003:45255, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:18,480 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:18,481 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-16T00:14:18,481 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:14:18,483 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-16T00:14:18,483 DEBUG [master/b878af803003:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-16T00:14:18,485 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-16T00:14:18,485 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:18,487 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-16T00:14:18,497 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36981 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:14:18,497 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43245 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:14:18,499 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 249cebb3b1696164ea2a3147502024d0, NAME => 'hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc
2024-12-16T00:14:18,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43245 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:14:18,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36981 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:14:18,507 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:14:18,507 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 249cebb3b1696164ea2a3147502024d0, disabling compactions & flushes
2024-12-16T00:14:18,507 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:14:18,507 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:14:18,507 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0. after waiting 0 ms
2024-12-16T00:14:18,507 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:14:18,507 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:14:18,508 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 249cebb3b1696164ea2a3147502024d0:

2024-12-16T00:14:18,509 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-16T00:14:18,509 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734308058509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734308058509"}]},"ts":"1734308058509"}
2024-12-16T00:14:18,512 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-16T00:14:18,513 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-16T00:14:18,513 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308058513"}]},"ts":"1734308058513"}
2024-12-16T00:14:18,515 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-16T00:14:18,529 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=249cebb3b1696164ea2a3147502024d0, ASSIGN}]
2024-12-16T00:14:18,532 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=249cebb3b1696164ea2a3147502024d0, ASSIGN
2024-12-16T00:14:18,533 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=249cebb3b1696164ea2a3147502024d0, ASSIGN; state=OFFLINE, location=b878af803003,41029,1734308057378; forceNewPlan=false, retain=false
2024-12-16T00:14:18,684 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=249cebb3b1696164ea2a3147502024d0, regionState=OPENING, regionLocation=b878af803003,41029,1734308057378
2024-12-16T00:14:18,690 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 249cebb3b1696164ea2a3147502024d0, server=b878af803003,41029,1734308057378}]
2024-12-16T00:14:18,847 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to b878af803003,41029,1734308057378
2024-12-16T00:14:18,856 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:14:18,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 249cebb3b1696164ea2a3147502024d0, NAME => 'hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:14:18,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 249cebb3b1696164ea2a3147502024d0
2024-12-16T00:14:18,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:14:18,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 249cebb3b1696164ea2a3147502024d0
2024-12-16T00:14:18,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 249cebb3b1696164ea2a3147502024d0
2024-12-16T00:14:18,858 INFO  [StoreOpener-249cebb3b1696164ea2a3147502024d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 249cebb3b1696164ea2a3147502024d0 
2024-12-16T00:14:18,860 INFO  [StoreOpener-249cebb3b1696164ea2a3147502024d0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 249cebb3b1696164ea2a3147502024d0 columnFamilyName info
2024-12-16T00:14:18,860 DEBUG [StoreOpener-249cebb3b1696164ea2a3147502024d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:18,860 INFO  [StoreOpener-249cebb3b1696164ea2a3147502024d0-1 {}] regionserver.HStore(327): Store=249cebb3b1696164ea2a3147502024d0/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:14:18,862 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/namespace/249cebb3b1696164ea2a3147502024d0
2024-12-16T00:14:18,862 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/namespace/249cebb3b1696164ea2a3147502024d0
2024-12-16T00:14:18,865 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 249cebb3b1696164ea2a3147502024d0
2024-12-16T00:14:18,868 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/namespace/249cebb3b1696164ea2a3147502024d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:14:18,869 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 249cebb3b1696164ea2a3147502024d0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=771276, jitterRate=-0.019272327423095703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:14:18,869 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 249cebb3b1696164ea2a3147502024d0:

2024-12-16T00:14:18,870 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0., pid=6, masterSystemTime=1734308058846
2024-12-16T00:14:18,871 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:14:18,872 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:14:18,872 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=249cebb3b1696164ea2a3147502024d0, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,41029,1734308057378
2024-12-16T00:14:18,876 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-16T00:14:18,876 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 249cebb3b1696164ea2a3147502024d0, server=b878af803003,41029,1734308057378 in 185 msec
2024-12-16T00:14:18,878 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-16T00:14:18,878 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=249cebb3b1696164ea2a3147502024d0, ASSIGN in 347 msec
2024-12-16T00:14:18,878 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-16T00:14:18,879 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308058879"}]},"ts":"1734308058879"}
2024-12-16T00:14:18,880 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-16T00:14:18,930 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-16T00:14:18,932 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-16T00:14:18,935 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 451 msec
2024-12-16T00:14:18,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:18,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:14:18,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:14:18,943 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-16T00:14:18,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:14:18,975 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 30 msec
2024-12-16T00:14:18,987 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-16T00:14:19,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:14:19,024 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 36 msec
2024-12-16T00:14:19,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-16T00:14:19,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-16T00:14:19,063 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.634sec
2024-12-16T00:14:19,063 INFO  [master/b878af803003:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-16T00:14:19,063 INFO  [master/b878af803003:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-16T00:14:19,063 INFO  [master/b878af803003:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-16T00:14:19,063 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-16T00:14:19,063 INFO  [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-16T00:14:19,063 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,45255,1734308057221-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:14:19,064 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,45255,1734308057221-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-16T00:14:19,067 DEBUG [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-16T00:14:19,067 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-16T00:14:19,067 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,45255,1734308057221-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:14:19,104 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x16a864b6 to 127.0.0.1:64341 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@477be5b7
2024-12-16T00:14:19,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce1748a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:14:19,116 DEBUG [hconnection-0x442f74fc-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:14:19,118 INFO  [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37596, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:14:19,120 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=b878af803003,45255,1734308057221
2024-12-16T00:14:19,120 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:14:19,125 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-16T00:14:19,125 INFO  [Time-limited test {}] wal.TestLogRolling(297): Starting testLogRollOnPipelineRestart
2024-12-16T00:14:19,125 INFO  [Time-limited test {}] wal.TestLogRolling(300): Replication=2
2024-12-16T00:14:19,126 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-16T00:14:19,128 INFO  [RS-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59808, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-16T00:14:19,129 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45255 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-16T00:14:19,129 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45255 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-16T00:14:19,130 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45255 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-16T00:14:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45255 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-12-16T00:14:19,132 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-12-16T00:14:19,132 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:19,132 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45255 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 9
2024-12-16T00:14:19,133 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-16T00:14:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45255 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-16T00:14:19,140 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36981 is added to blk_1073741837_1013 (size=395)
2024-12-16T00:14:19,140 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43245 is added to blk_1073741837_1013 (size=395)
2024-12-16T00:14:19,142 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => fac4dadd209cd543a61978f9f44aa6e8, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc
2024-12-16T00:14:19,148 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36981 is added to blk_1073741838_1014 (size=78)
2024-12-16T00:14:19,148 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43245 is added to blk_1073741838_1014 (size=78)
2024-12-16T00:14:19,149 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:14:19,149 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1681): Closing fac4dadd209cd543a61978f9f44aa6e8, disabling compactions & flushes
2024-12-16T00:14:19,149 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:14:19,149 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:14:19,149 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8. after waiting 0 ms
2024-12-16T00:14:19,149 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:14:19,149 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:14:19,149 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1635): Region close journal for fac4dadd209cd543a61978f9f44aa6e8:

2024-12-16T00:14:19,150 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-12-16T00:14:19,150 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1734308059150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734308059150"}]},"ts":"1734308059150"}
2024-12-16T00:14:19,152 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-16T00:14:19,153 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-16T00:14:19,153 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308059153"}]},"ts":"1734308059153"}
2024-12-16T00:14:19,154 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-12-16T00:14:19,171 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=fac4dadd209cd543a61978f9f44aa6e8, ASSIGN}]
2024-12-16T00:14:19,172 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=fac4dadd209cd543a61978f9f44aa6e8, ASSIGN
2024-12-16T00:14:19,173 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=fac4dadd209cd543a61978f9f44aa6e8, ASSIGN; state=OFFLINE, location=b878af803003,41029,1734308057378; forceNewPlan=false, retain=false
2024-12-16T00:14:19,324 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=fac4dadd209cd543a61978f9f44aa6e8, regionState=OPENING, regionLocation=b878af803003,41029,1734308057378
2024-12-16T00:14:19,329 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure fac4dadd209cd543a61978f9f44aa6e8, server=b878af803003,41029,1734308057378}]
2024-12-16T00:14:19,390 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:19,484 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to b878af803003,41029,1734308057378
2024-12-16T00:14:19,493 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:14:19,493 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => fac4dadd209cd543a61978f9f44aa6e8, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:14:19,493 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart fac4dadd209cd543a61978f9f44aa6e8
2024-12-16T00:14:19,494 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:14:19,494 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for fac4dadd209cd543a61978f9f44aa6e8
2024-12-16T00:14:19,494 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for fac4dadd209cd543a61978f9f44aa6e8
2024-12-16T00:14:19,496 INFO  [StoreOpener-fac4dadd209cd543a61978f9f44aa6e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region fac4dadd209cd543a61978f9f44aa6e8 
2024-12-16T00:14:19,498 INFO  [StoreOpener-fac4dadd209cd543a61978f9f44aa6e8-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fac4dadd209cd543a61978f9f44aa6e8 columnFamilyName info
2024-12-16T00:14:19,498 DEBUG [StoreOpener-fac4dadd209cd543a61978f9f44aa6e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:14:19,498 INFO  [StoreOpener-fac4dadd209cd543a61978f9f44aa6e8-1 {}] regionserver.HStore(327): Store=fac4dadd209cd543a61978f9f44aa6e8/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:14:19,499 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/default/TestLogRolling-testLogRollOnPipelineRestart/fac4dadd209cd543a61978f9f44aa6e8
2024-12-16T00:14:19,500 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/default/TestLogRolling-testLogRollOnPipelineRestart/fac4dadd209cd543a61978f9f44aa6e8
2024-12-16T00:14:19,503 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for fac4dadd209cd543a61978f9f44aa6e8
2024-12-16T00:14:19,505 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/default/TestLogRolling-testLogRollOnPipelineRestart/fac4dadd209cd543a61978f9f44aa6e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:14:19,505 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened fac4dadd209cd543a61978f9f44aa6e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=695952, jitterRate=-0.11505179107189178}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:14:19,506 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for fac4dadd209cd543a61978f9f44aa6e8:

2024-12-16T00:14:19,507 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8., pid=11, masterSystemTime=1734308059483
2024-12-16T00:14:19,509 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:14:19,509 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:14:19,509 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=fac4dadd209cd543a61978f9f44aa6e8, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,41029,1734308057378
2024-12-16T00:14:19,512 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-16T00:14:19,513 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure fac4dadd209cd543a61978f9f44aa6e8, server=b878af803003,41029,1734308057378 in 183 msec
2024-12-16T00:14:19,514 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-16T00:14:19,514 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=fac4dadd209cd543a61978f9f44aa6e8, ASSIGN in 342 msec
2024-12-16T00:14:19,515 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-16T00:14:19,515 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308059515"}]},"ts":"1734308059515"}
2024-12-16T00:14:19,517 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-12-16T00:14:19,555 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-12-16T00:14:19,557 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 425 msec
2024-12-16T00:14:20,391 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:21,393 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:22,371 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-16T00:14:22,388 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:22,388 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:22,388 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:22,389 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:22,389 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:22,389 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:22,392 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:22,394 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:14:22,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:23,395 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:23,750 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-12-16T00:14:24,396 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:25,397 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:26,398 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:26,862 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-16T00:14:26,862 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-12-16T00:14:26,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-16T00:14:26,864 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-16T00:14:27,400 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:28,402 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45255 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-16T00:14:29,135 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart, procId: 9 completed
2024-12-16T00:14:29,139 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-12-16T00:14:29,139 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:14:29,404 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:30,405 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:31,148 INFO  [Time-limited test {}] wal.TestLogRolling(337): log.getCurrentFileName()): hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886
2024-12-16T00:14:31,149 WARN  [ResponseProcessor for block BP-1291920219-172.17.0.2-1734308055567:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1291920219-172.17.0.2-1734308055567:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:31,149 WARN  [ResponseProcessor for block BP-1291920219-172.17.0.2-1734308055567:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1291920219-172.17.0.2-1734308055567:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1291920219-172.17.0.2-1734308055567:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:43245,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:31,149 WARN  [ResponseProcessor for block BP-1291920219-172.17.0.2-1734308055567:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1291920219-172.17.0.2-1734308055567:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-1291920219-172.17.0.2-1734308055567:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:43245,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:31,149 WARN  [DataStreamer for file /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221/b878af803003%2C45255%2C1734308057221.1734308057493 block BP-1291920219-172.17.0.2-1734308055567:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1291920219-172.17.0.2-1734308055567:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43245,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK], DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43245,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]) is bad.
2024-12-16T00:14:31,150 WARN  [DataStreamer for file /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886 block BP-1291920219-172.17.0.2-1734308055567:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1291920219-172.17.0.2-1734308055567:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK], DatanodeInfoWithStorage[127.0.0.1:43245,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43245,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]) is bad.
2024-12-16T00:14:31,150 WARN  [DataStreamer for file /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.meta.1734308058330.meta block BP-1291920219-172.17.0.2-1734308055567:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1291920219-172.17.0.2-1734308055567:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK], DatanodeInfoWithStorage[127.0.0.1:43245,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43245,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]) is bad.
2024-12-16T00:14:31,150 WARN  [PacketResponder: BP-1291920219-172.17.0.2-1734308055567:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:43245] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:31,150 WARN  [PacketResponder: BP-1291920219-172.17.0.2-1734308055567:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:43245] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:31,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2086059674_22 at /127.0.0.1:37182 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36981:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37182 dst: /127.0.0.1:36981
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:31,150 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1926291481_22 at /127.0.0.1:35124 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43245:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35124 dst: /127.0.0.1:43245
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:31,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2086059674_22 at /127.0.0.1:37166 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36981:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37166 dst: /127.0.0.1:36981
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:31,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2086059674_22 at /127.0.0.1:35180 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43245:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35180 dst: /127.0.0.1:43245
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:31,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2086059674_22 at /127.0.0.1:35154 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43245:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35154 dst: /127.0.0.1:43245
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:31,152 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1926291481_22 at /127.0.0.1:37132 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36981:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37132 dst: /127.0.0.1:36981
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:31,206 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1f1c6e43{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:31,207 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b10eec4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:14:31,207 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:14:31,207 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ffe559e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:14:31,208 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8f79290{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,STOPPED}
2024-12-16T00:14:31,210 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:14:31,210 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:14:31,211 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:14:31,211 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1291920219-172.17.0.2-1734308055567 (Datanode Uuid 9e9f74ac-5ae4-4b7e-89af-14153e85bce4) service to localhost/127.0.0.1:42703
2024-12-16T00:14:31,211 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data3/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:31,211 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data4/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:31,212 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:14:31,220 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:14:31,222 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:14:31,223 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:14:31,223 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:14:31,223 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:14:31,224 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cee7bf9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:14:31,224 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@428586a7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:14:31,313 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d14a030{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/java.io.tmpdir/jetty-localhost-37489-hadoop-hdfs-3_4_1-tests_jar-_-any-5286304674813900167/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:31,313 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@31d8a642{HTTP/1.1, (http/1.1)}{localhost:37489}
2024-12-16T00:14:31,313 INFO  [Time-limited test {}] server.Server(415): Started @189978ms
2024-12-16T00:14:31,314 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:14:31,329 WARN  [ResponseProcessor for block BP-1291920219-172.17.0.2-1734308055567:blk_1073741830_1017 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1291920219-172.17.0.2-1734308055567:blk_1073741830_1017
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:31,329 WARN  [ResponseProcessor for block BP-1291920219-172.17.0.2-1734308055567:blk_1073741833_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1291920219-172.17.0.2-1734308055567:blk_1073741833_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:31,329 WARN  [ResponseProcessor for block BP-1291920219-172.17.0.2-1734308055567:blk_1073741834_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1291920219-172.17.0.2-1734308055567:blk_1073741834_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:31,329 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1926291481_22 at /127.0.0.1:56060 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36981:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56060 dst: /127.0.0.1:36981
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:31,329 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2086059674_22 at /127.0.0.1:56044 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36981:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56044 dst: /127.0.0.1:36981
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:31,329 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2086059674_22 at /127.0.0.1:56046 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36981:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56046 dst: /127.0.0.1:36981
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:31,333 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@778a07eb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:31,333 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@578de26e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:14:31,333 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:14:31,334 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5dbd1f1f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:14:31,334 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b1885ce{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,STOPPED}
2024-12-16T00:14:31,335 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:14:31,335 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:14:31,335 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:14:31,335 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1291920219-172.17.0.2-1734308055567 (Datanode Uuid d9c191f8-4d5d-414a-a8eb-0f32d2637f45) service to localhost/127.0.0.1:42703
2024-12-16T00:14:31,335 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data1/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:31,335 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data2/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:31,335 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:14:31,344 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:14:31,347 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:14:31,348 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:14:31,348 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:14:31,348 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-16T00:14:31,349 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49c4a5a6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:14:31,349 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29028973{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:14:31,406 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:31,438 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@706a7986{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/java.io.tmpdir/jetty-localhost-34855-hadoop-hdfs-3_4_1-tests_jar-_-any-9292063125242518024/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:31,438 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@485ffef1{HTTP/1.1, (http/1.1)}{localhost:34855}
2024-12-16T00:14:31,439 INFO  [Time-limited test {}] server.Server(415): Started @190103ms
2024-12-16T00:14:31,440 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:14:31,642 WARN  [Thread-1101 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:14:31,645 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8ac414f3bf9c286a with lease ID 0x36234a392fa404c6: from storage DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1 node DatanodeRegistration(127.0.0.1:41465, datanodeUuid=9e9f74ac-5ae4-4b7e-89af-14153e85bce4, infoPort=39539, infoSecurePort=0, ipcPort=45075, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:14:31,645 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8ac414f3bf9c286a with lease ID 0x36234a392fa404c6: from storage DS-4b913d61-8138-4068-9ea5-08bb0f1bc574 node DatanodeRegistration(127.0.0.1:41465, datanodeUuid=9e9f74ac-5ae4-4b7e-89af-14153e85bce4, infoPort=39539, infoSecurePort=0, ipcPort=45075, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:14:31,874 WARN  [Thread-1121 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:14:31,877 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c65caf72ddec733 with lease ID 0x36234a392fa404c7: from storage DS-e5b6417b-1e48-469c-9939-0a3674fe0f94 node DatanodeRegistration(127.0.0.1:41275, datanodeUuid=d9c191f8-4d5d-414a-a8eb-0f32d2637f45, infoPort=42947, infoSecurePort=0, ipcPort=36791, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:14:31,877 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c65caf72ddec733 with lease ID 0x36234a392fa404c7: from storage DS-245570bb-64dc-4a1e-914c-ddc22a18bb81 node DatanodeRegistration(127.0.0.1:41275, datanodeUuid=d9c191f8-4d5d-414a-a8eb-0f32d2637f45, infoPort=42947, infoSecurePort=0, ipcPort=36791, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:14:32,407 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:32,458 INFO  [Time-limited test {}] wal.TestLogRolling(349): Data Nodes restarted
2024-12-16T00:14:32,461 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-12-16T00:14:32,464 WARN  [RS:0;b878af803003:41029.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=5, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:32,464 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C41029%2C1734308057378:(num 1734308057886) roll requested
2024-12-16T00:14:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41029 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:32,465 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C41029%2C1734308057378.1734308072465
2024-12-16T00:14:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41029 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:37596 deadline: 1734308082463, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL
2024-12-16T00:14:32,471 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886 newFile=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465
2024-12-16T00:14:32,471 WARN  [regionserver/b878af803003:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL
2024-12-16T00:14:32,471 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886 with entries=5, filesize=2.09 KB; new WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465
2024-12-16T00:14:32,472 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42947:42947),(127.0.0.1/127.0.0.1:39539:39539)]
2024-12-16T00:14:32,472 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886 is not closed yet, will try archiving it next time
2024-12-16T00:14:32,472 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:32,472 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:32,472 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886
2024-12-16T00:14:32,472 WARN  [IPC Server handler 3 on default port 42703 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1016
2024-12-16T00:14:32,473 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886 after 1ms
2024-12-16T00:14:33,409 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:34,410 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:34,646 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1016: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-16T00:14:35,411 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:36,412 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:36,474 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886 after 4002ms
2024-12-16T00:14:37,414 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:38,415 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:39,417 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:40,418 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:41,420 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:42,421 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:43,423 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:44,424 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:44,523 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-12-16T00:14:45,425 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:46,427 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:46,530 WARN  [ResponseProcessor for block BP-1291920219-172.17.0.2-1734308055567:blk_1073741839_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1291920219-172.17.0.2-1734308055567:blk_1073741839_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:46,531 WARN  [DataStreamer for file /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465 block BP-1291920219-172.17.0.2-1734308055567:blk_1073741839_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1291920219-172.17.0.2-1734308055567:blk_1073741839_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41275,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK], DatanodeInfoWithStorage[127.0.0.1:41465,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41275,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]) is bad.
2024-12-16T00:14:46,531 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2086059674_22 at /127.0.0.1:58490 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:41275:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58490 dst: /127.0.0.1:41275
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:46,532 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2086059674_22 at /127.0.0.1:34590 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:41465:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34590 dst: /127.0.0.1:41465
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:46,742 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@706a7986{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:46,742 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@485ffef1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:14:46,743 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:14:46,743 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29028973{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:14:46,743 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49c4a5a6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,STOPPED}
2024-12-16T00:14:46,747 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:14:46,747 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:14:46,747 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:14:46,747 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1291920219-172.17.0.2-1734308055567 (Datanode Uuid d9c191f8-4d5d-414a-a8eb-0f32d2637f45) service to localhost/127.0.0.1:42703
2024-12-16T00:14:46,748 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data1/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:46,748 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data2/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:46,748 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:14:46,757 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:14:46,761 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:14:46,762 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:14:46,762 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:14:46,762 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-16T00:14:46,763 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4223a417{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:14:46,764 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@81800f6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:14:46,857 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@221ebe8e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/java.io.tmpdir/jetty-localhost-43647-hadoop-hdfs-3_4_1-tests_jar-_-any-244098353448724226/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:46,857 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@47b64ea5{HTTP/1.1, (http/1.1)}{localhost:43647}
2024-12-16T00:14:46,857 INFO  [Time-limited test {}] server.Server(415): Started @205522ms
2024-12-16T00:14:46,858 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:14:46,876 WARN  [ResponseProcessor for block BP-1291920219-172.17.0.2-1734308055567:blk_1073741839_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1291920219-172.17.0.2-1734308055567:blk_1073741839_1020
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:46,876 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2086059674_22 at /127.0.0.1:48996 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:41465:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48996 dst: /127.0.0.1:41465
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:14:46,881 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7d14a030{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:46,881 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@31d8a642{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:14:46,881 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:14:46,881 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@428586a7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:14:46,882 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cee7bf9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,STOPPED}
2024-12-16T00:14:46,883 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:14:46,883 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:14:46,883 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1291920219-172.17.0.2-1734308055567 (Datanode Uuid 9e9f74ac-5ae4-4b7e-89af-14153e85bce4) service to localhost/127.0.0.1:42703
2024-12-16T00:14:46,883 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:14:46,883 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data3/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:46,884 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data4/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:14:46,884 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:14:46,894 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:14:46,896 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:14:46,897 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:14:46,897 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:14:46,897 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:14:46,897 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c71a368{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:14:46,897 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@721fa586{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:14:46,986 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64605d66{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/java.io.tmpdir/jetty-localhost-35241-hadoop-hdfs-3_4_1-tests_jar-_-any-2530161013515506544/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:14:46,987 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15314202{HTTP/1.1, (http/1.1)}{localhost:35241}
2024-12-16T00:14:46,987 INFO  [Time-limited test {}] server.Server(415): Started @205651ms
2024-12-16T00:14:46,988 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:14:47,180 WARN  [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(624): failed to reset thread pool timeout for FsDatasetAsyncDiskService
java.lang.NullPointerException: Cannot invoke "Object.getClass()" because "dataset" is null
	at org.apache.hadoop.hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer.run(HBaseTestingUtility.java:609) ~[test-classes/:2.7.0-SNAPSHOT]
2024-12-16T00:14:47,280 WARN  [Thread-1176 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:14:47,282 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd0b9a3547a87161e with lease ID 0x36234a392fa404c8: from storage DS-e5b6417b-1e48-469c-9939-0a3674fe0f94 node DatanodeRegistration(127.0.0.1:39423, datanodeUuid=d9c191f8-4d5d-414a-a8eb-0f32d2637f45, infoPort=40015, infoSecurePort=0, ipcPort=37991, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 8, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-16T00:14:47,282 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd0b9a3547a87161e with lease ID 0x36234a392fa404c8: from storage DS-245570bb-64dc-4a1e-914c-ddc22a18bb81 node DatanodeRegistration(127.0.0.1:39423, datanodeUuid=d9c191f8-4d5d-414a-a8eb-0f32d2637f45, infoPort=40015, infoSecurePort=0, ipcPort=37991, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:14:47,388 WARN  [Thread-1196 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:14:47,390 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x39cd4cea5ece3587 with lease ID 0x36234a392fa404c9: from storage DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1 node DatanodeRegistration(127.0.0.1:35161, datanodeUuid=9e9f74ac-5ae4-4b7e-89af-14153e85bce4, infoPort=46481, infoSecurePort=0, ipcPort=34685, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-16T00:14:47,390 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x39cd4cea5ece3587 with lease ID 0x36234a392fa404c9: from storage DS-4b913d61-8138-4068-9ea5-08bb0f1bc574 node DatanodeRegistration(127.0.0.1:35161, datanodeUuid=9e9f74ac-5ae4-4b7e-89af-14153e85bce4, infoPort=46481, infoSecurePort=0, ipcPort=34685, storageInfo=lv=-57;cid=testClusterID;nsid=1566150046;c=1734308055567), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:14:47,428 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:47,670 WARN  [master/b878af803003:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=95, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:47,671 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C45255%2C1734308057221:(num 1734308057493) roll requested
2024-12-16T00:14:47,671 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:47,672 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C45255%2C1734308057221.1734308087671
2024-12-16T00:14:47,672 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
	at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:47,683 WARN  [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
2024-12-16T00:14:47,683 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221/b878af803003%2C45255%2C1734308057221.1734308057493 with entries=92, filesize=46.00 KB; new WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221/b878af803003%2C45255%2C1734308057221.1734308087671
2024-12-16T00:14:47,683 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40015:40015),(127.0.0.1/127.0.0.1:46481:46481)]
2024-12-16T00:14:47,683 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221/b878af803003%2C45255%2C1734308057221.1734308057493 is not closed yet, will try archiving it next time
2024-12-16T00:14:47,683 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:47,683 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:47,683 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221/b878af803003%2C45255%2C1734308057221.1734308057493
2024-12-16T00:14:47,684 WARN  [IPC Server handler 2 on default port 42703 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221/b878af803003%2C45255%2C1734308057221.1734308057493 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741830_1017
2024-12-16T00:14:47,684 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221/b878af803003%2C45255%2C1734308057221.1734308057493 after 1ms
2024-12-16T00:14:48,004 INFO  [Time-limited test {}] wal.TestLogRolling(366): Data Nodes restarted
2024-12-16T00:14:48,007 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-12-16T00:14:48,010 WARN  [RS:0;b878af803003:41029.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=8, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41465,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:48,010 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C41029%2C1734308057378:(num 1734308072465) roll requested
2024-12-16T00:14:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41029 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41465,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:48,011 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C41029%2C1734308057378.1734308088010
2024-12-16T00:14:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41029 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:37596 deadline: 1734308098008, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL
2024-12-16T00:14:48,017 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465 newFile=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308088010
2024-12-16T00:14:48,018 WARN  [regionserver/b878af803003:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL
2024-12-16T00:14:48,018 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308088010
2024-12-16T00:14:48,018 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40015:40015),(127.0.0.1/127.0.0.1:46481:46481)]
2024-12-16T00:14:48,018 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465 is not closed yet, will try archiving it next time
2024-12-16T00:14:48,018 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41465,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:48,018 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41465,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:14:48,018 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465
2024-12-16T00:14:48,019 WARN  [IPC Server handler 4 on default port 42703 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465 has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741839_1020
2024-12-16T00:14:48,019 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465 after 1ms
2024-12-16T00:14:48,429 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:49,285 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741839_1020: GenerationStamp not matched, existing replica is blk_1073741839_1018
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-16T00:14:49,430 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:50,432 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:51,433 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:51,686 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221/b878af803003%2C45255%2C1734308057221.1734308057493 after 4003ms
2024-12-16T00:14:52,021 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465 after 4003ms
2024-12-16T00:14:52,392 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 2) replica(s):
0) Failed to delete replica blk_1073741830_1017: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-16T00:14:52,434 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:52,732 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:14:52,735 INFO  [RS-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54212, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:14:53,435 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:54,436 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:55,438 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:56,440 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:57,441 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:58,443 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:14:59,071 INFO  [master/b878af803003:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-16T00:14:59,071 INFO  [master/b878af803003:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-16T00:14:59,444 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:00,050 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C41029%2C1734308057378.1734308100049
2024-12-16T00:15:00,062 DEBUG [Time-limited test {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308088010 newFile=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049
2024-12-16T00:15:00,063 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308088010 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049
2024-12-16T00:15:00,063 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46481:46481),(127.0.0.1/127.0.0.1:40015:40015)]
2024-12-16T00:15:00,064 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308088010 is not closed yet, will try archiving it next time
2024-12-16T00:15:00,064 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886
2024-12-16T00:15:00,064 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886
2024-12-16T00:15:00,065 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886 after 0ms
2024-12-16T00:15:00,065 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886
2024-12-16T00:15:00,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741841_1023 (size=1264)
2024-12-16T00:15:00,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741841_1023 (size=1264)
2024-12-16T00:15:00,075 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1734308058869/Put/vlen=162/seqid=0]
2024-12-16T00:15:00,075 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [default/info:d/1734308058949/Put/vlen=9/seqid=0]
2024-12-16T00:15:00,075 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #5: [hbase/info:d/1734308058997/Put/vlen=7/seqid=0]
2024-12-16T00:15:00,075 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1734308059506/Put/vlen=218/seqid=0]
2024-12-16T00:15:00,075 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [row1002/info:/1734308069145/Put/vlen=1045/seqid=0]
2024-12-16T00:15:00,075 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308057886
2024-12-16T00:15:00,075 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465
2024-12-16T00:15:00,075 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465
2024-12-16T00:15:00,076 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465 after 1ms
2024-12-16T00:15:00,076 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465
2024-12-16T00:15:00,079 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #6: [row1003/info:/1734308082517/Put/vlen=1045/seqid=0]
2024-12-16T00:15:00,079 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #7: [row1004/info:/1734308084525/Put/vlen=1045/seqid=0]
2024-12-16T00:15:00,079 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308072465
2024-12-16T00:15:00,079 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308088010
2024-12-16T00:15:00,079 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308088010
2024-12-16T00:15:00,079 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308088010 after 0ms
2024-12-16T00:15:00,079 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308088010
2024-12-16T00:15:00,082 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #9: [row1005/info:/1734308098046/Put/vlen=1045/seqid=0]
2024-12-16T00:15:00,082 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049
2024-12-16T00:15:00,082 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049
2024-12-16T00:15:00,083 WARN  [IPC Server handler 2 on default port 42703 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049 has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741842_1025
2024-12-16T00:15:00,083 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049 after 1ms
2024-12-16T00:15:00,396 WARN  [ResponseProcessor for block BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:15:00,396 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1926291481_22 at /127.0.0.1:53748 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:35161:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53748 dst: /127.0.0.1:35161
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35161 remote=/127.0.0.1:53748]. Total timeout mills is 60000, 59665 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:15:00,397 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1926291481_22 at /127.0.0.1:36536 [Receiving block BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:39423:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36536 dst: /127.0.0.1:39423
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:15:00,397 WARN  [DataStreamer for file /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049 block BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35161,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK], DatanodeInfoWithStorage[127.0.0.1:39423,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35161,DS-eb45f1aa-41e1-4196-bb8a-9fa140b77bc1,DISK]) is bad.
2024-12-16T00:15:00,405 WARN  [DataStreamer for file /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049 block BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:15:00,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741842_1026 (size=85)
2024-12-16T00:15:00,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741842_1026 (size=85)
2024-12-16T00:15:00,446 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:01,447 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:02,448 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:03,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:04,085 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049 after 4003ms
2024-12-16T00:15:04,085 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049
2024-12-16T00:15:04,095 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049
2024-12-16T00:15:04,095 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.90 KB heapSize=5.42 KB
2024-12-16T00:15:04,095 WARN  [RS_OPEN_META-regionserver/b878af803003:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:15:04,096 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C41029%2C1734308057378.meta:.meta(num 1734308058330) roll requested
2024-12-16T00:15:04,096 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740:

2024-12-16T00:15:04,096 INFO  [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:15:04,096 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C41029%2C1734308057378.meta.1734308104096.meta
2024-12-16T00:15:04,096 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing fac4dadd209cd543a61978f9f44aa6e8 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-16T00:15:04,096 WARN  [RS:0;b878af803003:41029.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=11, requesting roll of WAL
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:15:04,097 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for fac4dadd209cd543a61978f9f44aa6e8:

2024-12-16T00:15:04,097 INFO  [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:15:04,097 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 249cebb3b1696164ea2a3147502024d0 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-16T00:15:04,098 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 249cebb3b1696164ea2a3147502024d0:

2024-12-16T00:15:04,098 INFO  [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:15:04,101 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-16T00:15:04,101 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-16T00:15:04,101 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x16a864b6 to 127.0.0.1:64341
2024-12-16T00:15:04,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:15:04,101 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-16T00:15:04,101 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=389352820, stopped=false
2024-12-16T00:15:04,101 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=b878af803003,45255,1734308057221
2024-12-16T00:15:04,102 WARN  [regionserver/b878af803003:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
2024-12-16T00:15:04,102 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.meta.1734308058330.meta with entries=11, filesize=3.66 KB; new WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.meta.1734308104096.meta
2024-12-16T00:15:04,102 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46481:46481),(127.0.0.1/127.0.0.1:40015:40015)]
2024-12-16T00:15:04,102 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.meta.1734308058330.meta is not closed yet, will try archiving it next time
2024-12-16T00:15:04,102 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog b878af803003%2C41029%2C1734308057378:(num 1734308100049) roll requested
2024-12-16T00:15:04,102 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:15:04,103 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36981,DS-e5b6417b-1e48-469c-9939-0a3674fe0f94,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:15:04,103 INFO  [regionserver/b878af803003:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C41029%2C1734308057378.1734308104102
2024-12-16T00:15:04,103 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.meta.1734308058330.meta
2024-12-16T00:15:04,103 WARN  [IPC Server handler 1 on default port 42703 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.meta.1734308058330.meta has not been closed. Lease recovery is in progress. RecoveryId = 1028 for block blk_1073741834_1015
2024-12-16T00:15:04,103 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.meta.1734308058330.meta after 0ms
2024-12-16T00:15:04,108 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049 newFile=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308104102
2024-12-16T00:15:04,108 WARN  [regionserver/b878af803003:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
2024-12-16T00:15:04,108 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308104102
2024-12-16T00:15:04,108 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40015:40015),(127.0.0.1/127.0.0.1:46481:46481)]
2024-12-16T00:15:04,108 DEBUG [regionserver/b878af803003:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049 is not closed yet, will try archiving it next time
2024-12-16T00:15:04,108 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:15:04,108 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1291920219-172.17.0.2-1734308055567:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-16T00:15:04,109 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049
2024-12-16T00:15:04,109 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049 after 0ms
2024-12-16T00:15:04,109 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.1734308100049 to hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/oldWALs/b878af803003%2C41029%2C1734308057378.1734308100049
2024-12-16T00:15:04,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:15:04,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:15:04,118 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-16T00:15:04,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:04,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:04,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:15:04,118 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,41029,1734308057378' *****
2024-12-16T00:15:04,118 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-16T00:15:04,118 INFO  [RS:0;b878af803003:41029 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-16T00:15:04,118 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:15:04,118 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:15:04,118 INFO  [RS:0;b878af803003:41029 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-16T00:15:04,118 INFO  [RS:0;b878af803003:41029 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-16T00:15:04,118 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-16T00:15:04,118 INFO  [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(3579): Received CLOSE for fac4dadd209cd543a61978f9f44aa6e8
2024-12-16T00:15:04,119 INFO  [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(3579): Received CLOSE for 249cebb3b1696164ea2a3147502024d0
2024-12-16T00:15:04,119 INFO  [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1224): stopping server b878af803003,41029,1734308057378
2024-12-16T00:15:04,119 DEBUG [RS:0;b878af803003:41029 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:15:04,119 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing fac4dadd209cd543a61978f9f44aa6e8, disabling compactions & flushes
2024-12-16T00:15:04,119 INFO  [RS:0;b878af803003:41029 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-16T00:15:04,119 INFO  [RS:0;b878af803003:41029 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-16T00:15:04,119 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:15:04,119 INFO  [RS:0;b878af803003:41029 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-16T00:15:04,119 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:15:04,119 INFO  [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-16T00:15:04,119 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8. after waiting 0 ms
2024-12-16T00:15:04,119 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:15:04,119 INFO  [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-16T00:15:04,119 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing fac4dadd209cd543a61978f9f44aa6e8 1/1 column families, dataSize=4.20 KB heapSize=4.98 KB
2024-12-16T00:15:04,120 DEBUG [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, fac4dadd209cd543a61978f9f44aa6e8=TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8., 249cebb3b1696164ea2a3147502024d0=hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.}
2024-12-16T00:15:04,120 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:15:04,120 DEBUG [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 249cebb3b1696164ea2a3147502024d0, fac4dadd209cd543a61978f9f44aa6e8
2024-12-16T00:15:04,120 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:15:04,120 WARN  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt?
2024-12-16T00:15:04,120 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:15:04,120 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:15:04,120 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:15:04,120 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.90 KB heapSize=5.89 KB
2024-12-16T00:15:04,120 WARN  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt?
2024-12-16T00:15:04,120 WARN  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt?
2024-12-16T00:15:04,135 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/default/TestLogRolling-testLogRollOnPipelineRestart/fac4dadd209cd543a61978f9f44aa6e8/.tmp/info/aeedc3c2d81a40898ed0001873263933 is 1080, key is row1002/info:/1734308069145/Put/seqid=0
2024-12-16T00:15:04,140 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741845_1030 (size=9270)
2024-12-16T00:15:04,140 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741845_1030 (size=9270)
2024-12-16T00:15:04,140 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/default/TestLogRolling-testLogRollOnPipelineRestart/fac4dadd209cd543a61978f9f44aa6e8/.tmp/info/aeedc3c2d81a40898ed0001873263933
2024-12-16T00:15:04,142 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/.tmp/info/76cb0a41223541568c6eb43f9e9df950 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8./info:regioninfo/1734308059509/Put/seqid=0
2024-12-16T00:15:04,147 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/default/TestLogRolling-testLogRollOnPipelineRestart/fac4dadd209cd543a61978f9f44aa6e8/.tmp/info/aeedc3c2d81a40898ed0001873263933 as hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/default/TestLogRolling-testLogRollOnPipelineRestart/fac4dadd209cd543a61978f9f44aa6e8/info/aeedc3c2d81a40898ed0001873263933
2024-12-16T00:15:04,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741846_1031 (size=8268)
2024-12-16T00:15:04,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741846_1031 (size=8268)
2024-12-16T00:15:04,148 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/.tmp/info/76cb0a41223541568c6eb43f9e9df950
2024-12-16T00:15:04,155 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/default/TestLogRolling-testLogRollOnPipelineRestart/fac4dadd209cd543a61978f9f44aa6e8/info/aeedc3c2d81a40898ed0001873263933, entries=4, sequenceid=12, filesize=9.1 K
2024-12-16T00:15:04,156 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for fac4dadd209cd543a61978f9f44aa6e8 in 37ms, sequenceid=12, compaction requested=false
2024-12-16T00:15:04,160 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/default/TestLogRolling-testLogRollOnPipelineRestart/fac4dadd209cd543a61978f9f44aa6e8/recovered.edits/15.seqid, newMaxSeqId=15, maxSeqId=1
2024-12-16T00:15:04,161 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:15:04,161 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for fac4dadd209cd543a61978f9f44aa6e8:

2024-12-16T00:15:04,161 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1734308059129.fac4dadd209cd543a61978f9f44aa6e8.
2024-12-16T00:15:04,161 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 249cebb3b1696164ea2a3147502024d0, disabling compactions & flushes
2024-12-16T00:15:04,161 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:15:04,161 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:15:04,161 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0. after waiting 0 ms
2024-12-16T00:15:04,162 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:15:04,162 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 249cebb3b1696164ea2a3147502024d0 1/1 column families, dataSize=78 B heapSize=728 B
2024-12-16T00:15:04,162 WARN  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt?
2024-12-16T00:15:04,167 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/.tmp/table/02f1a29a87a84857a924c930d2ce80c1 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1734308059515/Put/seqid=0
2024-12-16T00:15:04,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741847_1032 (size=5482)
2024-12-16T00:15:04,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741847_1032 (size=5482)
2024-12-16T00:15:04,172 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=244 B at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/.tmp/table/02f1a29a87a84857a924c930d2ce80c1
2024-12-16T00:15:04,177 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/namespace/249cebb3b1696164ea2a3147502024d0/.tmp/info/ad476d1ac83f4d29b734ae347ac7b6f2 is 45, key is default/info:d/1734308058949/Put/seqid=0
2024-12-16T00:15:04,178 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/.tmp/info/76cb0a41223541568c6eb43f9e9df950 as hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/info/76cb0a41223541568c6eb43f9e9df950
2024-12-16T00:15:04,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741848_1033 (size=5037)
2024-12-16T00:15:04,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741848_1033 (size=5037)
2024-12-16T00:15:04,182 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/namespace/249cebb3b1696164ea2a3147502024d0/.tmp/info/ad476d1ac83f4d29b734ae347ac7b6f2
2024-12-16T00:15:04,184 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/info/76cb0a41223541568c6eb43f9e9df950, entries=20, sequenceid=16, filesize=8.1 K
2024-12-16T00:15:04,185 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/.tmp/table/02f1a29a87a84857a924c930d2ce80c1 as hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/table/02f1a29a87a84857a924c930d2ce80c1
2024-12-16T00:15:04,189 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/namespace/249cebb3b1696164ea2a3147502024d0/.tmp/info/ad476d1ac83f4d29b734ae347ac7b6f2 as hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/namespace/249cebb3b1696164ea2a3147502024d0/info/ad476d1ac83f4d29b734ae347ac7b6f2
2024-12-16T00:15:04,191 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/table/02f1a29a87a84857a924c930d2ce80c1, entries=4, sequenceid=16, filesize=5.4 K
2024-12-16T00:15:04,192 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~2.90 KB/2972, heapSize ~5.14 KB/5264, currentSize=0 B/0 for 1588230740 in 72ms, sequenceid=16, compaction requested=false
2024-12-16T00:15:04,196 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/meta/1588230740/recovered.edits/19.seqid, newMaxSeqId=19, maxSeqId=1
2024-12-16T00:15:04,196 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/namespace/249cebb3b1696164ea2a3147502024d0/info/ad476d1ac83f4d29b734ae347ac7b6f2, entries=2, sequenceid=8, filesize=4.9 K
2024-12-16T00:15:04,196 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-16T00:15:04,196 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-16T00:15:04,197 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:15:04,197 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-16T00:15:04,197 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 249cebb3b1696164ea2a3147502024d0 in 35ms, sequenceid=8, compaction requested=false
2024-12-16T00:15:04,201 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/data/hbase/namespace/249cebb3b1696164ea2a3147502024d0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-12-16T00:15:04,202 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:15:04,202 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 249cebb3b1696164ea2a3147502024d0:

2024-12-16T00:15:04,202 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1734308058481.249cebb3b1696164ea2a3147502024d0.
2024-12-16T00:15:04,320 INFO  [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1250): stopping server b878af803003,41029,1734308057378; all regions closed.
2024-12-16T00:15:04,320 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378
2024-12-16T00:15:04,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741843_1027 (size=761)
2024-12-16T00:15:04,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741843_1027 (size=761)
2024-12-16T00:15:04,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:04,754 INFO  [regionserver/b878af803003:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-16T00:15:04,755 INFO  [regionserver/b878af803003:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-16T00:15:05,453 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:05,757 INFO  [regionserver/b878af803003:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:15:06,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:06,862 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-16T00:15:07,394 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1015: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-16T00:15:07,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:08,105 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.meta.1734308058330.meta after 4002ms
2024-12-16T00:15:08,106 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378/b878af803003%2C41029%2C1734308057378.meta.1734308058330.meta to hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/oldWALs/b878af803003%2C41029%2C1734308057378.meta.1734308058330.meta
2024-12-16T00:15:08,114 DEBUG [RS:0;b878af803003:41029 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/oldWALs
2024-12-16T00:15:08,114 INFO  [RS:0;b878af803003:41029 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog b878af803003%2C41029%2C1734308057378.meta:.meta(num 1734308104096)
2024-12-16T00:15:08,115 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/WALs/b878af803003,41029,1734308057378
2024-12-16T00:15:08,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741844_1029 (size=1979)
2024-12-16T00:15:08,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741844_1029 (size=1979)
2024-12-16T00:15:08,124 DEBUG [RS:0;b878af803003:41029 {}] wal.AbstractFSWAL(1071): Moved 4 WAL file(s) to /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/oldWALs
2024-12-16T00:15:08,124 INFO  [RS:0;b878af803003:41029 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog b878af803003%2C41029%2C1734308057378:(num 1734308104102)
2024-12-16T00:15:08,124 DEBUG [RS:0;b878af803003:41029 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:15:08,124 INFO  [RS:0;b878af803003:41029 {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:15:08,125 INFO  [RS:0;b878af803003:41029 {}] hbase.ChoreService(370): Chore service for: regionserver/b878af803003:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown
2024-12-16T00:15:08,125 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:15:08,125 INFO  [RS:0;b878af803003:41029 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:41029
2024-12-16T00:15:08,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:15:08,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b878af803003,41029,1734308057378
2024-12-16T00:15:08,184 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b878af803003,41029,1734308057378]
2024-12-16T00:15:08,185 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing b878af803003,41029,1734308057378; numProcessing=1
2024-12-16T00:15:08,193 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/b878af803003,41029,1734308057378 already deleted, retry=false
2024-12-16T00:15:08,193 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; b878af803003,41029,1734308057378 expired; onlineServers=0
2024-12-16T00:15:08,193 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,45255,1734308057221' *****
2024-12-16T00:15:08,193 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-16T00:15:08,194 DEBUG [M:0;b878af803003:45255 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d08812b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:15:08,194 INFO  [M:0;b878af803003:45255 {}] regionserver.HRegionServer(1224): stopping server b878af803003,45255,1734308057221
2024-12-16T00:15:08,194 INFO  [M:0;b878af803003:45255 {}] regionserver.HRegionServer(1250): stopping server b878af803003,45255,1734308057221; all regions closed.
2024-12-16T00:15:08,194 DEBUG [M:0;b878af803003:45255 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:15:08,194 DEBUG [M:0;b878af803003:45255 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-16T00:15:08,194 DEBUG [M:0;b878af803003:45255 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-16T00:15:08,194 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-16T00:15:08,194 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308057671 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308057671,5,FailOnTimeoutGroup]
2024-12-16T00:15:08,194 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308057671 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308057671,5,FailOnTimeoutGroup]
2024-12-16T00:15:08,195 INFO  [M:0;b878af803003:45255 {}] hbase.ChoreService(370): Chore service for: master/b878af803003:0 had [] on shutdown
2024-12-16T00:15:08,195 DEBUG [M:0;b878af803003:45255 {}] master.HMaster(1733): Stopping service threads
2024-12-16T00:15:08,195 INFO  [M:0;b878af803003:45255 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-16T00:15:08,196 INFO  [M:0;b878af803003:45255 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-16T00:15:08,196 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-16T00:15:08,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-16T00:15:08,204 DEBUG [M:0;b878af803003:45255 {}] zookeeper.ZKUtil(347): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-16T00:15:08,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:08,204 WARN  [M:0;b878af803003:45255 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-16T00:15:08,204 INFO  [M:0;b878af803003:45255 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-16T00:15:08,205 INFO  [M:0;b878af803003:45255 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-16T00:15:08,205 DEBUG [M:0;b878af803003:45255 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:15:08,205 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:15:08,205 INFO  [M:0;b878af803003:45255 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:15:08,205 DEBUG [M:0;b878af803003:45255 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:15:08,205 DEBUG [M:0;b878af803003:45255 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:15:08,205 DEBUG [M:0;b878af803003:45255 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:15:08,205 INFO  [M:0;b878af803003:45255 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.10 KB heapSize=49.26 KB
2024-12-16T00:15:08,219 DEBUG [M:0;b878af803003:45255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a59f80327738447d9d2147bb18d22e36 is 82, key is hbase:meta,,1/info:regioninfo/1734308058355/Put/seqid=0
2024-12-16T00:15:08,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741849_1034 (size=5672)
2024-12-16T00:15:08,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741849_1034 (size=5672)
2024-12-16T00:15:08,224 INFO  [M:0;b878af803003:45255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a59f80327738447d9d2147bb18d22e36
2024-12-16T00:15:08,241 DEBUG [M:0;b878af803003:45255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f8e1b048b784bad9ca8a1e98225fca0 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1734308059556/Put/seqid=0
2024-12-16T00:15:08,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741850_1035 (size=7470)
2024-12-16T00:15:08,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741850_1035 (size=7470)
2024-12-16T00:15:08,246 INFO  [M:0;b878af803003:45255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.50 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f8e1b048b784bad9ca8a1e98225fca0
2024-12-16T00:15:08,264 DEBUG [M:0;b878af803003:45255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8d76af524ac245628deda84623a27c48 is 69, key is b878af803003,41029,1734308057378/rs:state/1734308057736/Put/seqid=0
2024-12-16T00:15:08,269 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741851_1036 (size=5156)
2024-12-16T00:15:08,269 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741851_1036 (size=5156)
2024-12-16T00:15:08,269 INFO  [M:0;b878af803003:45255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8d76af524ac245628deda84623a27c48
2024-12-16T00:15:08,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:15:08,284 INFO  [RS:0;b878af803003:41029 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,41029,1734308057378; zookeeper connection closed.
2024-12-16T00:15:08,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41029-0x1002c2fecf20001, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:15:08,285 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4494114a {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4494114a
2024-12-16T00:15:08,285 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-16T00:15:08,286 DEBUG [M:0;b878af803003:45255 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1124092757a94834a89b7fb08fa15f68 is 52, key is load_balancer_on/state:d/1734308059122/Put/seqid=0
2024-12-16T00:15:08,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741852_1037 (size=5056)
2024-12-16T00:15:08,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741852_1037 (size=5056)
2024-12-16T00:15:08,291 INFO  [M:0;b878af803003:45255 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1124092757a94834a89b7fb08fa15f68
2024-12-16T00:15:08,296 DEBUG [M:0;b878af803003:45255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a59f80327738447d9d2147bb18d22e36 as hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a59f80327738447d9d2147bb18d22e36
2024-12-16T00:15:08,301 INFO  [M:0;b878af803003:45255 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a59f80327738447d9d2147bb18d22e36, entries=8, sequenceid=96, filesize=5.5 K
2024-12-16T00:15:08,302 DEBUG [M:0;b878af803003:45255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f8e1b048b784bad9ca8a1e98225fca0 as hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9f8e1b048b784bad9ca8a1e98225fca0
2024-12-16T00:15:08,307 INFO  [M:0;b878af803003:45255 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9f8e1b048b784bad9ca8a1e98225fca0, entries=11, sequenceid=96, filesize=7.3 K
2024-12-16T00:15:08,308 DEBUG [M:0;b878af803003:45255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8d76af524ac245628deda84623a27c48 as hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8d76af524ac245628deda84623a27c48
2024-12-16T00:15:08,314 INFO  [M:0;b878af803003:45255 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8d76af524ac245628deda84623a27c48, entries=1, sequenceid=96, filesize=5.0 K
2024-12-16T00:15:08,315 DEBUG [M:0;b878af803003:45255 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1124092757a94834a89b7fb08fa15f68 as hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1124092757a94834a89b7fb08fa15f68
2024-12-16T00:15:08,320 INFO  [M:0;b878af803003:45255 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:42703/user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1124092757a94834a89b7fb08fa15f68, entries=1, sequenceid=96, filesize=4.9 K
2024-12-16T00:15:08,321 INFO  [M:0;b878af803003:45255 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.10 KB/41064, heapSize ~49.20 KB/50376, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 115ms, sequenceid=96, compaction requested=false
2024-12-16T00:15:08,322 INFO  [M:0;b878af803003:45255 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:15:08,322 DEBUG [M:0;b878af803003:45255 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:15:08,322 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/dce75d0c-9173-836b-02d2-08ddfcca25bc/MasterData/WALs/b878af803003,45255,1734308057221
2024-12-16T00:15:08,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35161 is added to blk_1073741840_1021 (size=757)
2024-12-16T00:15:08,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39423 is added to blk_1073741840_1021 (size=757)
2024-12-16T00:15:08,325 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:15:08,325 INFO  [M:0;b878af803003:45255 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-16T00:15:08,325 INFO  [M:0;b878af803003:45255 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45255
2024-12-16T00:15:08,334 DEBUG [M:0;b878af803003:45255 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/b878af803003,45255,1734308057221 already deleted, retry=false
2024-12-16T00:15:08,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:15:08,442 INFO  [M:0;b878af803003:45255 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,45255,1734308057221; zookeeper connection closed.
2024-12-16T00:15:08,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45255-0x1002c2fecf20000, quorum=127.0.0.1:64341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:15:08,445 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64605d66{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:15:08,445 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15314202{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:15:08,445 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:15:08,445 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@721fa586{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:15:08,445 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c71a368{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,STOPPED}
2024-12-16T00:15:08,447 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:15:08,447 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:15:08,447 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:15:08,447 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1291920219-172.17.0.2-1734308055567 (Datanode Uuid 9e9f74ac-5ae4-4b7e-89af-14153e85bce4) service to localhost/127.0.0.1:42703
2024-12-16T00:15:08,447 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data3/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:15:08,448 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data4/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:15:08,448 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:15:08,450 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@221ebe8e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:15:08,450 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@47b64ea5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:15:08,450 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:15:08,450 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@81800f6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:15:08,450 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4223a417{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,STOPPED}
2024-12-16T00:15:08,452 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:15:08,452 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:15:08,452 WARN  [BP-1291920219-172.17.0.2-1734308055567 heartbeating to localhost/127.0.0.1:42703 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1291920219-172.17.0.2-1734308055567 (Datanode Uuid d9c191f8-4d5d-414a-a8eb-0f32d2637f45) service to localhost/127.0.0.1:42703
2024-12-16T00:15:08,452 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:15:08,452 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data1/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:15:08,453 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/cluster_cef765f6-84ad-34ae-c3a7-0e5b6bfbe687/dfs/data/data2/current/BP-1291920219-172.17.0.2-1734308055567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:15:08,453 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:15:08,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:08,459 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@56bd30de{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:15:08,459 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@377bc93b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:15:08,459 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:15:08,459 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@339df197{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:15:08,460 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@61b03158{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir/,STOPPED}
2024-12-16T00:15:08,465 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-16T00:15:08,481 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-16T00:15:08,490 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=99 (was 86)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42703
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-28-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (679588824) connection to localhost/127.0.0.1:42703 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (679588824) connection to localhost/127.0.0.1:42703 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-29-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-26-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-28-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-27-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42703
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-29-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (679588824) connection to localhost/127.0.0.1:42703 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42703
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-28-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-26-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-9-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-27-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42703
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:42703
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-27-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-29-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-26-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=450 (was 424) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=54 (was 33) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12041 (was 12134)
2024-12-16T00:15:08,497 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=99, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=54, ProcessCount=11, AvailableMemoryMB=12040
2024-12-16T00:15:08,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-16T00:15:08,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.log.dir so I do NOT create it in target/test-data/271d68ba-3276-9717-9536-72fc91545fdf
2024-12-16T00:15:08,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dd4512a5-209a-3836-cb90-425ed2ed276f/hadoop.tmp.dir so I do NOT create it in target/test-data/271d68ba-3276-9717-9536-72fc91545fdf
2024-12-16T00:15:08,497 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553, deleteOnExit=true
2024-12-16T00:15:08,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-16T00:15:08,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/test.cache.data in system properties and HBase conf
2024-12-16T00:15:08,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/hadoop.tmp.dir in system properties and HBase conf
2024-12-16T00:15:08,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/hadoop.log.dir in system properties and HBase conf
2024-12-16T00:15:08,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-16T00:15:08,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-16T00:15:08,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-16T00:15:08,498 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-16T00:15:08,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:15:08,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:15:08,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-16T00:15:08,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:15:08,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-16T00:15:08,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-16T00:15:08,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:15:08,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:15:08,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-16T00:15:08,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/nfs.dump.dir in system properties and HBase conf
2024-12-16T00:15:08,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/java.io.tmpdir in system properties and HBase conf
2024-12-16T00:15:08,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:15:08,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-16T00:15:08,499 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-16T00:15:08,511 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:15:08,804 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:15:08,808 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:15:08,816 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:15:08,816 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:15:08,816 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-16T00:15:08,816 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:15:08,817 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d004e71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:15:08,817 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1649f99b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:15:08,904 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@60255f8b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/java.io.tmpdir/jetty-localhost-34299-hadoop-hdfs-3_4_1-tests_jar-_-any-7667401348047867778/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:15:08,905 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e62c0ae{HTTP/1.1, (http/1.1)}{localhost:34299}
2024-12-16T00:15:08,905 INFO  [Time-limited test {}] server.Server(415): Started @227570ms
2024-12-16T00:15:08,915 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:15:09,123 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:15:09,126 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:15:09,127 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:15:09,127 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:15:09,127 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:15:09,127 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4884ca25{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:15:09,128 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75942022{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:15:09,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,179 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,179 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,183 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,183 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,183 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,185 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,190 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,190 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,190 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,191 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,229 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1242d2b6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/java.io.tmpdir/jetty-localhost-43165-hadoop-hdfs-3_4_1-tests_jar-_-any-2206421983039699252/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:15:09,229 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@56b096cd{HTTP/1.1, (http/1.1)}{localhost:43165}
2024-12-16T00:15:09,229 INFO  [Time-limited test {}] server.Server(415): Started @227894ms
2024-12-16T00:15:09,230 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:15:09,254 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-16T00:15:09,255 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,255 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,256 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,256 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,275 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,276 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,276 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,276 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,277 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,277 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,281 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,282 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,282 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,285 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:09,292 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:15:09,295 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:15:09,296 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:15:09,296 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:15:09,296 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-16T00:15:09,296 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57736e47{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:15:09,297 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3271ec70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:15:09,387 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@723a5a8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/java.io.tmpdir/jetty-localhost-38455-hadoop-hdfs-3_4_1-tests_jar-_-any-8269880929504087658/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:15:09,387 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4333752{HTTP/1.1, (http/1.1)}{localhost:38455}
2024-12-16T00:15:09,387 INFO  [Time-limited test {}] server.Server(415): Started @228052ms
2024-12-16T00:15:09,389 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:15:09,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:10,159 WARN  [Thread-1414 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553/dfs/data/data1/current/BP-1194759261-172.17.0.2-1734308108522/current, will proceed with Du for space computation calculation, 
2024-12-16T00:15:10,160 WARN  [Thread-1415 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553/dfs/data/data2/current/BP-1194759261-172.17.0.2-1734308108522/current, will proceed with Du for space computation calculation, 
2024-12-16T00:15:10,179 WARN  [Thread-1377 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:15:10,181 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80e2ee25889812fa with lease ID 0xc0018d35a703e3c4: Processing first storage report for DS-7bf6dfbe-6ec0-40ca-98b9-9fd6f2162970 from datanode DatanodeRegistration(127.0.0.1:39165, datanodeUuid=78c86dbb-836a-4101-b32f-f044bcda4986, infoPort=41257, infoSecurePort=0, ipcPort=44149, storageInfo=lv=-57;cid=testClusterID;nsid=1750641463;c=1734308108522)
2024-12-16T00:15:10,181 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80e2ee25889812fa with lease ID 0xc0018d35a703e3c4: from storage DS-7bf6dfbe-6ec0-40ca-98b9-9fd6f2162970 node DatanodeRegistration(127.0.0.1:39165, datanodeUuid=78c86dbb-836a-4101-b32f-f044bcda4986, infoPort=41257, infoSecurePort=0, ipcPort=44149, storageInfo=lv=-57;cid=testClusterID;nsid=1750641463;c=1734308108522), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:15:10,181 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80e2ee25889812fa with lease ID 0xc0018d35a703e3c4: Processing first storage report for DS-77b8ee7b-99e7-4700-8ba2-33731b1dd563 from datanode DatanodeRegistration(127.0.0.1:39165, datanodeUuid=78c86dbb-836a-4101-b32f-f044bcda4986, infoPort=41257, infoSecurePort=0, ipcPort=44149, storageInfo=lv=-57;cid=testClusterID;nsid=1750641463;c=1734308108522)
2024-12-16T00:15:10,181 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80e2ee25889812fa with lease ID 0xc0018d35a703e3c4: from storage DS-77b8ee7b-99e7-4700-8ba2-33731b1dd563 node DatanodeRegistration(127.0.0.1:39165, datanodeUuid=78c86dbb-836a-4101-b32f-f044bcda4986, infoPort=41257, infoSecurePort=0, ipcPort=44149, storageInfo=lv=-57;cid=testClusterID;nsid=1750641463;c=1734308108522), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:15:10,270 WARN  [Thread-1426 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553/dfs/data/data4/current/BP-1194759261-172.17.0.2-1734308108522/current, will proceed with Du for space computation calculation, 
2024-12-16T00:15:10,270 WARN  [Thread-1425 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553/dfs/data/data3/current/BP-1194759261-172.17.0.2-1734308108522/current, will proceed with Du for space computation calculation, 
2024-12-16T00:15:10,284 WARN  [Thread-1401 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:15:10,285 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4c7e0e5225599669 with lease ID 0xc0018d35a703e3c5: Processing first storage report for DS-dd2e2222-0565-4cc8-9528-d1f838b4139f from datanode DatanodeRegistration(127.0.0.1:46609, datanodeUuid=459e13cd-ac40-4e4e-ab63-c40244732e37, infoPort=43821, infoSecurePort=0, ipcPort=41541, storageInfo=lv=-57;cid=testClusterID;nsid=1750641463;c=1734308108522)
2024-12-16T00:15:10,286 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4c7e0e5225599669 with lease ID 0xc0018d35a703e3c5: from storage DS-dd2e2222-0565-4cc8-9528-d1f838b4139f node DatanodeRegistration(127.0.0.1:46609, datanodeUuid=459e13cd-ac40-4e4e-ab63-c40244732e37, infoPort=43821, infoSecurePort=0, ipcPort=41541, storageInfo=lv=-57;cid=testClusterID;nsid=1750641463;c=1734308108522), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:15:10,286 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4c7e0e5225599669 with lease ID 0xc0018d35a703e3c5: Processing first storage report for DS-05316152-93d6-48fd-b907-b5ea2aceea40 from datanode DatanodeRegistration(127.0.0.1:46609, datanodeUuid=459e13cd-ac40-4e4e-ab63-c40244732e37, infoPort=43821, infoSecurePort=0, ipcPort=41541, storageInfo=lv=-57;cid=testClusterID;nsid=1750641463;c=1734308108522)
2024-12-16T00:15:10,286 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4c7e0e5225599669 with lease ID 0xc0018d35a703e3c5: from storage DS-05316152-93d6-48fd-b907-b5ea2aceea40 node DatanodeRegistration(127.0.0.1:46609, datanodeUuid=459e13cd-ac40-4e4e-ab63-c40244732e37, infoPort=43821, infoSecurePort=0, ipcPort=41541, storageInfo=lv=-57;cid=testClusterID;nsid=1750641463;c=1734308108522), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:15:10,333 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf
2024-12-16T00:15:10,338 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553/zookeeper_0, clientPort=53278, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-16T00:15:10,339 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=53278
2024-12-16T00:15:10,339 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:15:10,340 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:15:10,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:15:10,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:15:10,348 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577 with version=8
2024-12-16T00:15:10,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/hbase-staging
2024-12-16T00:15:10,350 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/b878af803003:0 server-side Connection retries=45
2024-12-16T00:15:10,350 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:15:10,350 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:15:10,350 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:15:10,350 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:15:10,350 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:15:10,351 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:15:10,351 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:15:10,351 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:42851
2024-12-16T00:15:10,352 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:15:10,353 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:15:10,355 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:42851 connecting to ZooKeeper ensemble=127.0.0.1:53278
2024-12-16T00:15:10,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428510x0, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:15:10,423 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42851-0x1002c30bc810000 connected
2024-12-16T00:15:10,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:10,489 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:15:10,490 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:15:10,491 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:15:10,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42851
2024-12-16T00:15:10,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42851
2024-12-16T00:15:10,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42851
2024-12-16T00:15:10,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42851
2024-12-16T00:15:10,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42851
2024-12-16T00:15:10,494 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577, hbase.cluster.distributed=false
2024-12-16T00:15:10,514 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/b878af803003:0 server-side Connection retries=45
2024-12-16T00:15:10,514 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:15:10,514 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:15:10,514 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:15:10,514 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:15:10,515 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:15:10,515 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:15:10,515 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:15:10,515 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36481
2024-12-16T00:15:10,515 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-16T00:15:10,516 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-16T00:15:10,516 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:15:10,518 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:15:10,520 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:36481 connecting to ZooKeeper ensemble=127.0.0.1:53278
2024-12-16T00:15:10,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:364810x0, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:15:10,530 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36481-0x1002c30bc810001 connected
2024-12-16T00:15:10,531 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:15:10,531 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:15:10,532 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:15:10,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36481
2024-12-16T00:15:10,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36481
2024-12-16T00:15:10,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36481
2024-12-16T00:15:10,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36481
2024-12-16T00:15:10,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36481
2024-12-16T00:15:10,533 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/b878af803003,42851,1734308110350
2024-12-16T00:15:10,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:15:10,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:15:10,539 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b878af803003,42851,1734308110350
2024-12-16T00:15:10,546 DEBUG [M:0;b878af803003:42851 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b878af803003:42851
2024-12-16T00:15:10,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:15:10,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:15:10,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:10,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:10,547 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:15:10,547 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b878af803003,42851,1734308110350 from backup master directory
2024-12-16T00:15:10,548 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:15:10,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b878af803003,42851,1734308110350
2024-12-16T00:15:10,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:15:10,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:15:10,555 WARN  [master/b878af803003:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:15:10,555 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b878af803003,42851,1734308110350
2024-12-16T00:15:10,564 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:15:10,565 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:15:10,566 DEBUG [master/b878af803003:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/hbase.id with ID: 12604fa5-2438-4a22-8366-13f21f78758e
2024-12-16T00:15:10,579 INFO  [master/b878af803003:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:15:10,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:10,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:10,595 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:15:10,595 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:15:10,596 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-16T00:15:10,597 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-16T00:15:10,597 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:15:10,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:15:10,604 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:15:10,605 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store
2024-12-16T00:15:10,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:15:10,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:15:10,612 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:15:10,612 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:15:10,612 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:15:10,612 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:15:10,612 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:15:10,612 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:15:10,612 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:15:10,612 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:15:10,613 WARN  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/.initializing
2024-12-16T00:15:10,613 DEBUG [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/WALs/b878af803003,42851,1734308110350
2024-12-16T00:15:10,615 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C42851%2C1734308110350, suffix=, logDir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/WALs/b878af803003,42851,1734308110350, archiveDir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/oldWALs, maxLogs=10
2024-12-16T00:15:10,616 INFO  [master/b878af803003:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C42851%2C1734308110350.1734308110616
2024-12-16T00:15:10,621 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/WALs/b878af803003,42851,1734308110350/b878af803003%2C42851%2C1734308110350.1734308110616
2024-12-16T00:15:10,621 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43821:43821),(127.0.0.1/127.0.0.1:41257:41257)]
2024-12-16T00:15:10,621 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:15:10,621 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:15:10,621 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:15:10,621 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:15:10,623 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:15:10,624 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-16T00:15:10,624 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:10,625 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:15:10,625 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:15:10,626 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-16T00:15:10,626 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:10,627 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:15:10,627 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:15:10,628 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-16T00:15:10,628 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:10,629 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:15:10,629 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:15:10,630 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-16T00:15:10,630 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:10,631 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:15:10,631 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:15:10,632 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:15:10,634 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-16T00:15:10,635 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:15:10,637 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:15:10,637 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=760268, jitterRate=-0.03326970338821411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-16T00:15:10,637 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:15:10,638 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-16T00:15:10,640 DEBUG [master/b878af803003:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43c3e27f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:15:10,641 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-16T00:15:10,641 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-16T00:15:10,642 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-16T00:15:10,642 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-16T00:15:10,642 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-16T00:15:10,642 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-16T00:15:10,642 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-16T00:15:10,644 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-16T00:15:10,644 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-16T00:15:10,654 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-16T00:15:10,654 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-16T00:15:10,655 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-16T00:15:10,663 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-16T00:15:10,664 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-16T00:15:10,665 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-16T00:15:10,672 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-16T00:15:10,672 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-16T00:15:10,680 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-16T00:15:10,682 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-16T00:15:10,688 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-16T00:15:10,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:15:10,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:15:10,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:10,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:10,698 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=b878af803003,42851,1734308110350, sessionid=0x1002c30bc810000, setting cluster-up flag (Was=false)
2024-12-16T00:15:10,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:10,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:10,739 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-16T00:15:10,740 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,42851,1734308110350
2024-12-16T00:15:10,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:10,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:10,780 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-16T00:15:10,781 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,42851,1734308110350
2024-12-16T00:15:10,784 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-16T00:15:10,784 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-16T00:15:10,784 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-16T00:15:10,784 DEBUG [master/b878af803003:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b878af803003,42851,1734308110350
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-16T00:15:10,784 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:15:10,784 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:15:10,784 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:15:10,784 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:15:10,785 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b878af803003:0, corePoolSize=10, maxPoolSize=10
2024-12-16T00:15:10,785 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,785 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:15:10,785 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,785 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734308140785
2024-12-16T00:15:10,786 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-16T00:15:10,786 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-16T00:15:10,786 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-16T00:15:10,786 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-16T00:15:10,786 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-16T00:15:10,786 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-16T00:15:10,786 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:10,786 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:15:10,786 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-16T00:15:10,786 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-16T00:15:10,786 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-16T00:15:10,786 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-16T00:15:10,787 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-16T00:15:10,787 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-16T00:15:10,787 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308110787,5,FailOnTimeoutGroup]
2024-12-16T00:15:10,787 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:10,787 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:15:10,790 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308110787,5,FailOnTimeoutGroup]
2024-12-16T00:15:10,790 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:10,790 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-16T00:15:10,790 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:10,791 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:10,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:15:10,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:15:10,798 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-16T00:15:10,798 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577
2024-12-16T00:15:10,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:15:10,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:15:10,805 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:15:10,807 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:15:10,808 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:15:10,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:10,808 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:15:10,808 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:15:10,809 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:15:10,809 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:10,810 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:15:10,810 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:15:10,811 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:15:10,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:10,811 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:15:10,812 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740
2024-12-16T00:15:10,812 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740
2024-12-16T00:15:10,814 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:15:10,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:15:10,817 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:15:10,817 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=731030, jitterRate=-0.0704481303691864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:15:10,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:15:10,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:15:10,817 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:15:10,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:15:10,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:15:10,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:15:10,818 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-16T00:15:10,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:15:10,819 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:15:10,819 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-16T00:15:10,819 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-16T00:15:10,820 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-16T00:15:10,821 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-16T00:15:10,843 DEBUG [RS:0;b878af803003:36481 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b878af803003:36481
2024-12-16T00:15:10,844 INFO  [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1008): ClusterId : 12604fa5-2438-4a22-8366-13f21f78758e
2024-12-16T00:15:10,844 DEBUG [RS:0;b878af803003:36481 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-16T00:15:10,855 DEBUG [RS:0;b878af803003:36481 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-16T00:15:10,855 DEBUG [RS:0;b878af803003:36481 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-16T00:15:10,864 DEBUG [RS:0;b878af803003:36481 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-16T00:15:10,865 DEBUG [RS:0;b878af803003:36481 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a87d033, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:15:10,865 DEBUG [RS:0;b878af803003:36481 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43468d16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:15:10,865 INFO  [RS:0;b878af803003:36481 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-16T00:15:10,865 INFO  [RS:0;b878af803003:36481 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-16T00:15:10,865 DEBUG [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-16T00:15:10,866 INFO  [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(3073): reportForDuty to master=b878af803003,42851,1734308110350 with isa=b878af803003/172.17.0.2:36481, startcode=1734308110514
2024-12-16T00:15:10,866 DEBUG [RS:0;b878af803003:36481 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:15:10,868 INFO  [RS-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36611, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:15:10,868 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42851 {}] master.ServerManager(332): Checking decommissioned status of RegionServer b878af803003,36481,1734308110514
2024-12-16T00:15:10,868 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42851 {}] master.ServerManager(486): Registering regionserver=b878af803003,36481,1734308110514
2024-12-16T00:15:10,869 DEBUG [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577
2024-12-16T00:15:10,870 DEBUG [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:34017
2024-12-16T00:15:10,870 DEBUG [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-16T00:15:10,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:15:10,881 DEBUG [RS:0;b878af803003:36481 {}] zookeeper.ZKUtil(111): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b878af803003,36481,1734308110514
2024-12-16T00:15:10,881 WARN  [RS:0;b878af803003:36481 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:15:10,881 INFO  [RS:0;b878af803003:36481 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:15:10,881 DEBUG [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514
2024-12-16T00:15:10,881 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b878af803003,36481,1734308110514]
2024-12-16T00:15:10,884 DEBUG [RS:0;b878af803003:36481 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-16T00:15:10,884 INFO  [RS:0;b878af803003:36481 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-16T00:15:10,885 INFO  [RS:0;b878af803003:36481 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-16T00:15:10,886 INFO  [RS:0;b878af803003:36481 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-16T00:15:10,886 INFO  [RS:0;b878af803003:36481 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:10,886 INFO  [RS:0;b878af803003:36481 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-16T00:15:10,887 INFO  [RS:0;b878af803003:36481 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:15:10,887 DEBUG [RS:0;b878af803003:36481 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:15:10,890 INFO  [RS:0;b878af803003:36481 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:10,890 INFO  [RS:0;b878af803003:36481 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:10,890 INFO  [RS:0;b878af803003:36481 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:10,890 INFO  [RS:0;b878af803003:36481 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:10,890 INFO  [RS:0;b878af803003:36481 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,36481,1734308110514-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:15:10,902 INFO  [RS:0;b878af803003:36481 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-16T00:15:10,902 INFO  [RS:0;b878af803003:36481 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,36481,1734308110514-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:10,914 INFO  [RS:0;b878af803003:36481 {}] regionserver.Replication(204): b878af803003,36481,1734308110514 started
2024-12-16T00:15:10,914 INFO  [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1767): Serving as b878af803003,36481,1734308110514, RpcServer on b878af803003/172.17.0.2:36481, sessionid=0x1002c30bc810001
2024-12-16T00:15:10,915 DEBUG [RS:0;b878af803003:36481 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-16T00:15:10,915 DEBUG [RS:0;b878af803003:36481 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b878af803003,36481,1734308110514
2024-12-16T00:15:10,915 DEBUG [RS:0;b878af803003:36481 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,36481,1734308110514'
2024-12-16T00:15:10,915 DEBUG [RS:0;b878af803003:36481 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-16T00:15:10,915 DEBUG [RS:0;b878af803003:36481 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-16T00:15:10,916 DEBUG [RS:0;b878af803003:36481 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-16T00:15:10,916 DEBUG [RS:0;b878af803003:36481 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-16T00:15:10,916 DEBUG [RS:0;b878af803003:36481 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b878af803003,36481,1734308110514
2024-12-16T00:15:10,916 DEBUG [RS:0;b878af803003:36481 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,36481,1734308110514'
2024-12-16T00:15:10,916 DEBUG [RS:0;b878af803003:36481 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-16T00:15:10,916 DEBUG [RS:0;b878af803003:36481 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-16T00:15:10,916 DEBUG [RS:0;b878af803003:36481 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-16T00:15:10,916 INFO  [RS:0;b878af803003:36481 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-16T00:15:10,916 INFO  [RS:0;b878af803003:36481 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-16T00:15:10,971 WARN  [b878af803003:42851 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-16T00:15:11,020 INFO  [RS:0;b878af803003:36481 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C36481%2C1734308110514, suffix=, logDir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514, archiveDir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/oldWALs, maxLogs=32
2024-12-16T00:15:11,021 INFO  [RS:0;b878af803003:36481 {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C36481%2C1734308110514.1734308111021
2024-12-16T00:15:11,032 INFO  [RS:0;b878af803003:36481 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308111021
2024-12-16T00:15:11,032 DEBUG [RS:0;b878af803003:36481 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41257:41257),(127.0.0.1/127.0.0.1:43821:43821)]
2024-12-16T00:15:11,221 DEBUG [b878af803003:42851 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-16T00:15:11,221 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b878af803003,36481,1734308110514
2024-12-16T00:15:11,223 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,36481,1734308110514, state=OPENING
2024-12-16T00:15:11,230 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-16T00:15:11,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:11,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:11,239 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=b878af803003,36481,1734308110514}]
2024-12-16T00:15:11,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:15:11,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:15:11,393 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to b878af803003,36481,1734308110514
2024-12-16T00:15:11,394 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-16T00:15:11,397 INFO  [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57964, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-16T00:15:11,403 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-16T00:15:11,403 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:15:11,405 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C36481%2C1734308110514.meta, suffix=.meta, logDir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514, archiveDir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/oldWALs, maxLogs=32
2024-12-16T00:15:11,405 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C36481%2C1734308110514.meta.1734308111405.meta
2024-12-16T00:15:11,411 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.meta.1734308111405.meta
2024-12-16T00:15:11,411 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41257:41257),(127.0.0.1/127.0.0.1:43821:43821)]
2024-12-16T00:15:11,411 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:15:11,411 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-16T00:15:11,411 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-16T00:15:11,411 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-16T00:15:11,411 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-16T00:15:11,411 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:15:11,412 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-16T00:15:11,412 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-16T00:15:11,413 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:15:11,414 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:15:11,414 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:11,414 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:15:11,414 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:15:11,415 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:15:11,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:11,415 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:15:11,415 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:15:11,416 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:15:11,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:11,417 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:15:11,417 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740
2024-12-16T00:15:11,418 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740
2024-12-16T00:15:11,420 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:15:11,422 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:15:11,422 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=832773, jitterRate=0.05892680585384369}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:15:11,422 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:15:11,423 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734308111393
2024-12-16T00:15:11,425 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-16T00:15:11,425 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-16T00:15:11,426 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,36481,1734308110514
2024-12-16T00:15:11,427 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,36481,1734308110514, state=OPEN
2024-12-16T00:15:11,458 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:11,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:15:11,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:15:11,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:15:11,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:15:11,469 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-16T00:15:11,469 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=b878af803003,36481,1734308110514 in 225 msec
2024-12-16T00:15:11,474 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-16T00:15:11,474 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 650 msec
2024-12-16T00:15:11,478 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 692 msec
2024-12-16T00:15:11,478 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734308111478, completionTime=-1
2024-12-16T00:15:11,479 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-16T00:15:11,479 DEBUG [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-16T00:15:11,480 DEBUG [hconnection-0x72bad880-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:15:11,482 INFO  [RS-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57972, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:15:11,482 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-16T00:15:11,483 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734308171483
2024-12-16T00:15:11,483 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734308231483
2024-12-16T00:15:11,483 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec
2024-12-16T00:15:11,505 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42851,1734308110350-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:11,506 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42851,1734308110350-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:11,506 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42851,1734308110350-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:11,506 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b878af803003:42851, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:11,506 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:11,506 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-16T00:15:11,506 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:15:11,507 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-16T00:15:11,508 DEBUG [master/b878af803003:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-16T00:15:11,509 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-16T00:15:11,509 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:11,510 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-16T00:15:11,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:15:11,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:15:11,523 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 590e4a408d4f83e289cb88cefe762b5c, NAME => 'hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577
2024-12-16T00:15:11,530 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:15:11,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:15:11,532 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:15:11,532 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 590e4a408d4f83e289cb88cefe762b5c, disabling compactions & flushes
2024-12-16T00:15:11,532 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:15:11,532 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:15:11,532 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c. after waiting 0 ms
2024-12-16T00:15:11,532 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:15:11,532 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:15:11,532 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 590e4a408d4f83e289cb88cefe762b5c:

2024-12-16T00:15:11,534 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-16T00:15:11,534 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734308111534"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734308111534"}]},"ts":"1734308111534"}
2024-12-16T00:15:11,536 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-16T00:15:11,537 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-16T00:15:11,538 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308111538"}]},"ts":"1734308111538"}
2024-12-16T00:15:11,539 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-16T00:15:11,555 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=590e4a408d4f83e289cb88cefe762b5c, ASSIGN}]
2024-12-16T00:15:11,556 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=590e4a408d4f83e289cb88cefe762b5c, ASSIGN
2024-12-16T00:15:11,557 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=590e4a408d4f83e289cb88cefe762b5c, ASSIGN; state=OFFLINE, location=b878af803003,36481,1734308110514; forceNewPlan=false, retain=false
2024-12-16T00:15:11,708 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=590e4a408d4f83e289cb88cefe762b5c, regionState=OPENING, regionLocation=b878af803003,36481,1734308110514
2024-12-16T00:15:11,713 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 590e4a408d4f83e289cb88cefe762b5c, server=b878af803003,36481,1734308110514}]
2024-12-16T00:15:11,868 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to b878af803003,36481,1734308110514
2024-12-16T00:15:11,874 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:15:11,874 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 590e4a408d4f83e289cb88cefe762b5c, NAME => 'hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:15:11,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 590e4a408d4f83e289cb88cefe762b5c
2024-12-16T00:15:11,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:15:11,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 590e4a408d4f83e289cb88cefe762b5c
2024-12-16T00:15:11,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 590e4a408d4f83e289cb88cefe762b5c
2024-12-16T00:15:11,877 INFO  [StoreOpener-590e4a408d4f83e289cb88cefe762b5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 590e4a408d4f83e289cb88cefe762b5c 
2024-12-16T00:15:11,880 INFO  [StoreOpener-590e4a408d4f83e289cb88cefe762b5c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 590e4a408d4f83e289cb88cefe762b5c columnFamilyName info
2024-12-16T00:15:11,880 DEBUG [StoreOpener-590e4a408d4f83e289cb88cefe762b5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:11,880 INFO  [StoreOpener-590e4a408d4f83e289cb88cefe762b5c-1 {}] regionserver.HStore(327): Store=590e4a408d4f83e289cb88cefe762b5c/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:15:11,881 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/namespace/590e4a408d4f83e289cb88cefe762b5c
2024-12-16T00:15:11,881 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/namespace/590e4a408d4f83e289cb88cefe762b5c
2024-12-16T00:15:11,883 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 590e4a408d4f83e289cb88cefe762b5c
2024-12-16T00:15:11,886 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/namespace/590e4a408d4f83e289cb88cefe762b5c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:15:11,886 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 590e4a408d4f83e289cb88cefe762b5c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=821991, jitterRate=0.04521608352661133}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:15:11,886 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 590e4a408d4f83e289cb88cefe762b5c:

2024-12-16T00:15:11,887 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c., pid=6, masterSystemTime=1734308111868
2024-12-16T00:15:11,889 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:15:11,889 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:15:11,890 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=590e4a408d4f83e289cb88cefe762b5c, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,36481,1734308110514
2024-12-16T00:15:11,893 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-16T00:15:11,894 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 590e4a408d4f83e289cb88cefe762b5c, server=b878af803003,36481,1734308110514 in 178 msec
2024-12-16T00:15:11,895 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-16T00:15:11,895 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=590e4a408d4f83e289cb88cefe762b5c, ASSIGN in 338 msec
2024-12-16T00:15:11,896 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-16T00:15:11,896 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308111896"}]},"ts":"1734308111896"}
2024-12-16T00:15:11,898 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-16T00:15:11,906 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-16T00:15:11,908 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 400 msec
2024-12-16T00:15:11,909 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-16T00:15:11,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:15:11,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:11,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:15:11,980 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-16T00:15:11,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:15:12,010 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 28 msec
2024-12-16T00:15:12,013 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-16T00:15:12,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:15:12,043 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 28 msec
2024-12-16T00:15:12,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-16T00:15:12,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-16T00:15:12,080 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.525sec
2024-12-16T00:15:12,080 INFO  [master/b878af803003:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-16T00:15:12,080 INFO  [master/b878af803003:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-16T00:15:12,080 INFO  [master/b878af803003:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-16T00:15:12,080 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-16T00:15:12,081 INFO  [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-16T00:15:12,081 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42851,1734308110350-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:15:12,081 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42851,1734308110350-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-16T00:15:12,082 DEBUG [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-16T00:15:12,082 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-16T00:15:12,083 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42851,1734308110350-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:15:12,137 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31a91180 to 127.0.0.1:53278 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@201faa3b
2024-12-16T00:15:12,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d21be41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:15:12,150 DEBUG [hconnection-0x5446b7d0-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:15:12,153 INFO  [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57984, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:15:12,156 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=b878af803003,42851,1734308110350
2024-12-16T00:15:12,156 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:15:12,158 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-16T00:15:12,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-16T00:15:12,161 INFO  [RS-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34122, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-16T00:15:12,162 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-16T00:15:12,162 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-16T00:15:12,162 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-16T00:15:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:15:12,164 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-16T00:15:12,164 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:12,164 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 9
2024-12-16T00:15:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-16T00:15:12,165 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-16T00:15:12,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741837_1013 (size=405)
2024-12-16T00:15:12,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741837_1013 (size=405)
2024-12-16T00:15:12,173 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d57841ff468eaa4432d9a357c67b4a37, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577
2024-12-16T00:15:12,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741838_1014 (size=88)
2024-12-16T00:15:12,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741838_1014 (size=88)
2024-12-16T00:15:12,180 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:15:12,180 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1681): Closing d57841ff468eaa4432d9a357c67b4a37, disabling compactions & flushes
2024-12-16T00:15:12,180 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:12,180 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:12,180 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37. after waiting 0 ms
2024-12-16T00:15:12,180 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:12,180 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:12,180 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for d57841ff468eaa4432d9a357c67b4a37:

2024-12-16T00:15:12,181 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-16T00:15:12,182 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1734308112182"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734308112182"}]},"ts":"1734308112182"}
2024-12-16T00:15:12,183 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-16T00:15:12,184 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-16T00:15:12,185 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308112184"}]},"ts":"1734308112184"}
2024-12-16T00:15:12,186 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-12-16T00:15:12,205 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=d57841ff468eaa4432d9a357c67b4a37, ASSIGN}]
2024-12-16T00:15:12,207 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=d57841ff468eaa4432d9a357c67b4a37, ASSIGN
2024-12-16T00:15:12,208 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=d57841ff468eaa4432d9a357c67b4a37, ASSIGN; state=OFFLINE, location=b878af803003,36481,1734308110514; forceNewPlan=false, retain=false
2024-12-16T00:15:12,358 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=d57841ff468eaa4432d9a357c67b4a37, regionState=OPENING, regionLocation=b878af803003,36481,1734308110514
2024-12-16T00:15:12,361 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure d57841ff468eaa4432d9a357c67b4a37, server=b878af803003,36481,1734308110514}]
2024-12-16T00:15:12,459 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:12,516 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to b878af803003,36481,1734308110514
2024-12-16T00:15:12,523 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:12,524 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => d57841ff468eaa4432d9a357c67b4a37, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:15:12,524 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling d57841ff468eaa4432d9a357c67b4a37
2024-12-16T00:15:12,524 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:15:12,524 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for d57841ff468eaa4432d9a357c67b4a37
2024-12-16T00:15:12,524 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for d57841ff468eaa4432d9a357c67b4a37
2024-12-16T00:15:12,526 INFO  [StoreOpener-d57841ff468eaa4432d9a357c67b4a37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d57841ff468eaa4432d9a357c67b4a37 
2024-12-16T00:15:12,527 INFO  [StoreOpener-d57841ff468eaa4432d9a357c67b4a37-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d57841ff468eaa4432d9a357c67b4a37 columnFamilyName info
2024-12-16T00:15:12,528 DEBUG [StoreOpener-d57841ff468eaa4432d9a357c67b4a37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:15:12,528 INFO  [StoreOpener-d57841ff468eaa4432d9a357c67b4a37-1 {}] regionserver.HStore(327): Store=d57841ff468eaa4432d9a357c67b4a37/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:15:12,529 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37
2024-12-16T00:15:12,530 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37
2024-12-16T00:15:12,532 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for d57841ff468eaa4432d9a357c67b4a37
2024-12-16T00:15:12,536 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:15:12,536 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened d57841ff468eaa4432d9a357c67b4a37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=712516, jitterRate=-0.09398908913135529}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:15:12,537 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for d57841ff468eaa4432d9a357c67b4a37:

2024-12-16T00:15:12,538 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37., pid=11, masterSystemTime=1734308112516
2024-12-16T00:15:12,540 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:12,540 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:12,541 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=d57841ff468eaa4432d9a357c67b4a37, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,36481,1734308110514
2024-12-16T00:15:12,545 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-16T00:15:12,545 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure d57841ff468eaa4432d9a357c67b4a37, server=b878af803003,36481,1734308110514 in 182 msec
2024-12-16T00:15:12,547 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-16T00:15:12,547 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=d57841ff468eaa4432d9a357c67b4a37, ASSIGN in 340 msec
2024-12-16T00:15:12,548 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-16T00:15:12,548 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308112548"}]},"ts":"1734308112548"}
2024-12-16T00:15:12,550 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-12-16T00:15:12,598 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-16T00:15:12,601 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 436 msec
2024-12-16T00:15:13,461 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:14,462 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:15,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:16,464 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:16,916 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-16T00:15:16,917 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,917 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,917 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,917 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,935 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,935 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,935 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,936 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,936 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,936 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,938 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,939 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,939 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,941 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:15:16,945 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:15:16,945 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-12-16T00:15:16,946 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-16T00:15:16,946 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-16T00:15:16,947 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-12-16T00:15:17,465 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:17,466 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta after 68083ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor199.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:15:18,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:19,468 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:20,470 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:21,471 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-16T00:15:22,168 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 9 completed
2024-12-16T00:15:22,172 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:15:22,172 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:22,182 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush hbase:namespace
2024-12-16T00:15:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace
2024-12-16T00:15:22,188 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_PREPARE
2024-12-16T00:15:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12
2024-12-16T00:15:22,189 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-16T00:15:22,191 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-16T00:15:22,356 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to b878af803003,36481,1734308110514
2024-12-16T00:15:22,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36481 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13
2024-12-16T00:15:22,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:15:22,358 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2837): Flushing 590e4a408d4f83e289cb88cefe762b5c 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-16T00:15:22,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/namespace/590e4a408d4f83e289cb88cefe762b5c/.tmp/info/0ae8a22f15074e62ad577f66e59c5a5e is 45, key is default/info:d/1734308111986/Put/seqid=0
2024-12-16T00:15:22,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741839_1015 (size=5037)
2024-12-16T00:15:22,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741839_1015 (size=5037)
2024-12-16T00:15:22,384 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/namespace/590e4a408d4f83e289cb88cefe762b5c/.tmp/info/0ae8a22f15074e62ad577f66e59c5a5e
2024-12-16T00:15:22,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/namespace/590e4a408d4f83e289cb88cefe762b5c/.tmp/info/0ae8a22f15074e62ad577f66e59c5a5e as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/namespace/590e4a408d4f83e289cb88cefe762b5c/info/0ae8a22f15074e62ad577f66e59c5a5e
2024-12-16T00:15:22,396 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/namespace/590e4a408d4f83e289cb88cefe762b5c/info/0ae8a22f15074e62ad577f66e59c5a5e, entries=2, sequenceid=6, filesize=4.9 K
2024-12-16T00:15:22,397 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 590e4a408d4f83e289cb88cefe762b5c in 39ms, sequenceid=6, compaction requested=false
2024-12-16T00:15:22,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2538): Flush status journal for 590e4a408d4f83e289cb88cefe762b5c:

2024-12-16T00:15:22,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:15:22,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=13
2024-12-16T00:15:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.HMaster(4106): Remote procedure done, pid=13
2024-12-16T00:15:22,402 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12
2024-12-16T00:15:22,402 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 210 msec
2024-12-16T00:15:22,404 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace in 220 msec
2024-12-16T00:15:22,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:23,473 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:24,475 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:25,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:26,478 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:27,479 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:28,480 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:29,482 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:30,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:31,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12
2024-12-16T00:15:32,192 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: hbase:namespace, procId: 12 completed
2024-12-16T00:15:32,207 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:15:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] procedure2.ProcedureExecutor(1098): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:15:32,210 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-16T00:15:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14
2024-12-16T00:15:32,211 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-16T00:15:32,211 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-16T00:15:32,363 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to b878af803003,36481,1734308110514
2024-12-16T00:15:32,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36481 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15
2024-12-16T00:15:32,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:32,365 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2837): Flushing d57841ff468eaa4432d9a357c67b4a37 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-16T00:15:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/273a1c2467264ada9dd23063795252ce is 1080, key is row0001/info:/1734308132200/Put/seqid=0
2024-12-16T00:15:32,391 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741840_1016 (size=6033)
2024-12-16T00:15:32,391 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741840_1016 (size=6033)
2024-12-16T00:15:32,391 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/273a1c2467264ada9dd23063795252ce
2024-12-16T00:15:32,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/273a1c2467264ada9dd23063795252ce as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/273a1c2467264ada9dd23063795252ce
2024-12-16T00:15:32,402 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/273a1c2467264ada9dd23063795252ce, entries=1, sequenceid=5, filesize=5.9 K
2024-12-16T00:15:32,403 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for d57841ff468eaa4432d9a357c67b4a37 in 38ms, sequenceid=5, compaction requested=false
2024-12-16T00:15:32,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2538): Flush status journal for d57841ff468eaa4432d9a357c67b4a37:

2024-12-16T00:15:32,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:32,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15
2024-12-16T00:15:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.HMaster(4106): Remote procedure done, pid=15
2024-12-16T00:15:32,406 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=14
2024-12-16T00:15:32,406 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 194 msec
2024-12-16T00:15:32,408 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 200 msec
2024-12-16T00:15:32,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:33,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:34,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:35,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:36,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:37,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:38,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:39,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:40,332 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-16T00:15:40,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:41,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14
2024-12-16T00:15:42,213 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 14 completed
2024-12-16T00:15:42,224 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:15:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] procedure2.ProcedureExecutor(1098): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:15:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16
2024-12-16T00:15:42,228 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-16T00:15:42,229 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-16T00:15:42,229 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-16T00:15:42,382 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to b878af803003,36481,1734308110514
2024-12-16T00:15:42,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36481 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17
2024-12-16T00:15:42,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:42,385 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2837): Flushing d57841ff468eaa4432d9a357c67b4a37 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-16T00:15:42,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/b66def675ade4c1a82fd8945c53a02cf is 1080, key is row0002/info:/1734308142215/Put/seqid=0
2024-12-16T00:15:42,400 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741841_1017 (size=6033)
2024-12-16T00:15:42,401 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741841_1017 (size=6033)
2024-12-16T00:15:42,401 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/b66def675ade4c1a82fd8945c53a02cf
2024-12-16T00:15:42,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/b66def675ade4c1a82fd8945c53a02cf as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/b66def675ade4c1a82fd8945c53a02cf
2024-12-16T00:15:42,413 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/b66def675ade4c1a82fd8945c53a02cf, entries=1, sequenceid=9, filesize=5.9 K
2024-12-16T00:15:42,414 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for d57841ff468eaa4432d9a357c67b4a37 in 30ms, sequenceid=9, compaction requested=false
2024-12-16T00:15:42,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2538): Flush status journal for d57841ff468eaa4432d9a357c67b4a37:

2024-12-16T00:15:42,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:42,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=17
2024-12-16T00:15:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.HMaster(4106): Remote procedure done, pid=17
2024-12-16T00:15:42,418 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16
2024-12-16T00:15:42,418 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 187 msec
2024-12-16T00:15:42,420 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 194 msec
2024-12-16T00:15:42,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:43,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:44,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:44,865 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:15:44,869 INFO  [RS-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41782, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:15:45,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:46,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:47,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:48,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:49,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:50,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:51,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16
2024-12-16T00:15:52,231 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 16 completed
2024-12-16T00:15:52,236 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C36481%2C1734308110514.1734308152235
2024-12-16T00:15:52,247 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308111021 with entries=13, filesize=6.41 KB; new WAL /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308152235
2024-12-16T00:15:52,247 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41257:41257),(127.0.0.1/127.0.0.1:43821:43821)]
2024-12-16T00:15:52,247 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308111021 is not closed yet, will try archiving it next time
2024-12-16T00:15:52,249 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741833_1009 (size=6574)
2024-12-16T00:15:52,249 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741833_1009 (size=6574)
2024-12-16T00:15:52,252 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:15:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:15:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18
2024-12-16T00:15:52,254 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-16T00:15:52,254 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-16T00:15:52,255 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-16T00:15:52,408 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to b878af803003,36481,1734308110514
2024-12-16T00:15:52,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36481 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19
2024-12-16T00:15:52,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:52,410 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2837): Flushing d57841ff468eaa4432d9a357c67b4a37 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-16T00:15:52,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/9850a7633e3e4807969a4346cd275b2b is 1080, key is row0003/info:/1734308152233/Put/seqid=0
2024-12-16T00:15:52,426 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741843_1019 (size=6033)
2024-12-16T00:15:52,427 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741843_1019 (size=6033)
2024-12-16T00:15:52,428 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/9850a7633e3e4807969a4346cd275b2b
2024-12-16T00:15:52,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/9850a7633e3e4807969a4346cd275b2b as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/9850a7633e3e4807969a4346cd275b2b
2024-12-16T00:15:52,444 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/9850a7633e3e4807969a4346cd275b2b, entries=1, sequenceid=13, filesize=5.9 K
2024-12-16T00:15:52,445 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for d57841ff468eaa4432d9a357c67b4a37 in 35ms, sequenceid=13, compaction requested=true
2024-12-16T00:15:52,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2538): Flush status journal for d57841ff468eaa4432d9a357c67b4a37:

2024-12-16T00:15:52,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:15:52,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19
2024-12-16T00:15:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.HMaster(4106): Remote procedure done, pid=19
2024-12-16T00:15:52,448 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18
2024-12-16T00:15:52,448 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 191 msec
2024-12-16T00:15:52,450 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 197 msec
2024-12-16T00:15:52,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:52,714 INFO  [master/b878af803003:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-16T00:15:52,714 INFO  [master/b878af803003:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-16T00:15:53,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:54,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:55,516 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:56,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:56,875 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 590e4a408d4f83e289cb88cefe762b5c, had cached 0 bytes from a total of 5037
2024-12-16T00:15:57,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:57,524 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region d57841ff468eaa4432d9a357c67b4a37, had cached 0 bytes from a total of 18099
2024-12-16T00:15:58,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:15:59,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:00,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:01,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18
2024-12-16T00:16:02,257 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 18 completed
2024-12-16T00:16:02,258 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-16T00:16:02,262 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-16T00:16:02,262 DEBUG [Time-limited test {}] regionserver.HStore(1540): d57841ff468eaa4432d9a357c67b4a37/info is initiating minor compaction (all files)
2024-12-16T00:16:02,262 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-16T00:16:02,263 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:02,263 INFO  [Time-limited test {}] regionserver.HRegion(2351): Starting compaction of d57841ff468eaa4432d9a357c67b4a37/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:16:02,263 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/273a1c2467264ada9dd23063795252ce, hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/b66def675ade4c1a82fd8945c53a02cf, hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/9850a7633e3e4807969a4346cd275b2b] into tmpdir=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp, totalSize=17.7 K
2024-12-16T00:16:02,264 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 273a1c2467264ada9dd23063795252ce, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1734308132200
2024-12-16T00:16:02,265 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting b66def675ade4c1a82fd8945c53a02cf, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1734308142215
2024-12-16T00:16:02,266 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 9850a7633e3e4807969a4346cd275b2b, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1734308152233
2024-12-16T00:16:02,282 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): d57841ff468eaa4432d9a357c67b4a37#info#compaction#32 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:16:02,283 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/f089702ed96e4687aa19bfdb97670034 is 1080, key is row0001/info:/1734308132200/Put/seqid=0
2024-12-16T00:16:02,288 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741844_1020 (size=8296)
2024-12-16T00:16:02,289 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741844_1020 (size=8296)
2024-12-16T00:16:02,294 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/f089702ed96e4687aa19bfdb97670034 as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/f089702ed96e4687aa19bfdb97670034
2024-12-16T00:16:02,300 INFO  [Time-limited test {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d57841ff468eaa4432d9a357c67b4a37/info of d57841ff468eaa4432d9a357c67b4a37 into f089702ed96e4687aa19bfdb97670034(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-16T00:16:02,300 DEBUG [Time-limited test {}] regionserver.HRegion(2381): Compaction status journal for d57841ff468eaa4432d9a357c67b4a37:

2024-12-16T00:16:02,301 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C36481%2C1734308110514.1734308162301
2024-12-16T00:16:02,307 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308152235 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308162301
2024-12-16T00:16:02,307 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43821:43821),(127.0.0.1/127.0.0.1:41257:41257)]
2024-12-16T00:16:02,308 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308152235 is not closed yet, will try archiving it next time
2024-12-16T00:16:02,308 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308111021 to hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/oldWALs/b878af803003%2C36481%2C1734308110514.1734308111021
2024-12-16T00:16:02,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741842_1018 (size=2520)
2024-12-16T00:16:02,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741842_1018 (size=2520)
2024-12-16T00:16:02,311 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:16:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] procedure2.ProcedureExecutor(1098): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:16:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20
2024-12-16T00:16:02,313 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-16T00:16:02,314 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-16T00:16:02,314 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-16T00:16:02,465 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to b878af803003,36481,1734308110514
2024-12-16T00:16:02,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36481 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21
2024-12-16T00:16:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:16:02,467 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2837): Flushing d57841ff468eaa4432d9a357c67b4a37 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-16T00:16:02,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/7c7b5f913bcb4178a7393bae65eeb094 is 1080, key is row0000/info:/1734308162300/Put/seqid=0
2024-12-16T00:16:02,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741846_1022 (size=6033)
2024-12-16T00:16:02,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741846_1022 (size=6033)
2024-12-16T00:16:02,481 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/7c7b5f913bcb4178a7393bae65eeb094
2024-12-16T00:16:02,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/7c7b5f913bcb4178a7393bae65eeb094 as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/7c7b5f913bcb4178a7393bae65eeb094
2024-12-16T00:16:02,491 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/7c7b5f913bcb4178a7393bae65eeb094, entries=1, sequenceid=18, filesize=5.9 K
2024-12-16T00:16:02,492 INFO  [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for d57841ff468eaa4432d9a357c67b4a37 in 25ms, sequenceid=18, compaction requested=false
2024-12-16T00:16:02,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2538): Flush status journal for d57841ff468eaa4432d9a357c67b4a37:

2024-12-16T00:16:02,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:16:02,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b878af803003:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=21
2024-12-16T00:16:02,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.HMaster(4106): Remote procedure done, pid=21
2024-12-16T00:16:02,496 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=21, resume processing ppid=20
2024-12-16T00:16:02,496 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 180 msec
2024-12-16T00:16:02,497 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 185 msec
2024-12-16T00:16:02,525 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:03,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:04,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:05,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:06,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:07,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:08,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:09,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:10,333 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-16T00:16:10,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:11,510 DEBUG [master/b878af803003:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 590e4a408d4f83e289cb88cefe762b5c changed from -1.0 to 0.0, refreshing cache
2024-12-16T00:16:11,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42851 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20
2024-12-16T00:16:12,316 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 20 completed
2024-12-16T00:16:12,321 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C36481%2C1734308110514.1734308172321
2024-12-16T00:16:12,333 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308162301 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308172321
2024-12-16T00:16:12,333 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41257:41257),(127.0.0.1/127.0.0.1:43821:43821)]
2024-12-16T00:16:12,333 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308162301 is not closed yet, will try archiving it next time
2024-12-16T00:16:12,333 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514/b878af803003%2C36481%2C1734308110514.1734308152235 to hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/oldWALs/b878af803003%2C36481%2C1734308110514.1734308152235
2024-12-16T00:16:12,333 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-16T00:16:12,333 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-16T00:16:12,334 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x31a91180 to 127.0.0.1:53278
2024-12-16T00:16:12,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:16:12,334 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-16T00:16:12,334 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=378962139, stopped=false
2024-12-16T00:16:12,334 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=b878af803003,42851,1734308110350
2024-12-16T00:16:12,335 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741845_1021 (size=2026)
2024-12-16T00:16:12,335 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741845_1021 (size=2026)
2024-12-16T00:16:12,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:16:12,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:16:12,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:12,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:12,345 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-16T00:16:12,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:16:12,345 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,36481,1734308110514' *****
2024-12-16T00:16:12,345 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-16T00:16:12,345 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:16:12,345 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:16:12,346 INFO  [RS:0;b878af803003:36481 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-16T00:16:12,346 INFO  [RS:0;b878af803003:36481 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-16T00:16:12,346 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-16T00:16:12,346 INFO  [RS:0;b878af803003:36481 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-16T00:16:12,346 INFO  [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(3579): Received CLOSE for d57841ff468eaa4432d9a357c67b4a37
2024-12-16T00:16:12,346 INFO  [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(3579): Received CLOSE for 590e4a408d4f83e289cb88cefe762b5c
2024-12-16T00:16:12,346 INFO  [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1224): stopping server b878af803003,36481,1734308110514
2024-12-16T00:16:12,346 DEBUG [RS:0;b878af803003:36481 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:16:12,346 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d57841ff468eaa4432d9a357c67b4a37, disabling compactions & flushes
2024-12-16T00:16:12,346 INFO  [RS:0;b878af803003:36481 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-16T00:16:12,346 INFO  [RS:0;b878af803003:36481 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-16T00:16:12,346 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:16:12,347 INFO  [RS:0;b878af803003:36481 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-16T00:16:12,347 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:16:12,347 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37. after waiting 0 ms
2024-12-16T00:16:12,347 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:16:12,347 INFO  [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-16T00:16:12,347 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d57841ff468eaa4432d9a357c67b4a37 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-16T00:16:12,347 INFO  [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-16T00:16:12,347 DEBUG [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1603): Online Regions={d57841ff468eaa4432d9a357c67b4a37=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37., 590e4a408d4f83e289cb88cefe762b5c=hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c., 1588230740=hbase:meta,,1.1588230740}
2024-12-16T00:16:12,347 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:16:12,347 DEBUG [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 590e4a408d4f83e289cb88cefe762b5c, d57841ff468eaa4432d9a357c67b4a37
2024-12-16T00:16:12,347 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:16:12,347 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:16:12,347 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:16:12,347 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:16:12,347 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=3.05 KB heapSize=5.55 KB
2024-12-16T00:16:12,351 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/050356fcd6a6414895fa9e3ba5e8241e is 1080, key is row0001/info:/1734308172318/Put/seqid=0
2024-12-16T00:16:12,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741848_1024 (size=6033)
2024-12-16T00:16:12,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741848_1024 (size=6033)
2024-12-16T00:16:12,357 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/050356fcd6a6414895fa9e3ba5e8241e
2024-12-16T00:16:12,362 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/.tmp/info/050356fcd6a6414895fa9e3ba5e8241e as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/050356fcd6a6414895fa9e3ba5e8241e
2024-12-16T00:16:12,367 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/050356fcd6a6414895fa9e3ba5e8241e, entries=1, sequenceid=22, filesize=5.9 K
2024-12-16T00:16:12,368 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for d57841ff468eaa4432d9a357c67b4a37 in 21ms, sequenceid=22, compaction requested=true
2024-12-16T00:16:12,369 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/273a1c2467264ada9dd23063795252ce, hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/b66def675ade4c1a82fd8945c53a02cf, hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/9850a7633e3e4807969a4346cd275b2b] to archive
2024-12-16T00:16:12,369 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/.tmp/info/a10566e4a1e241ec9b8817d4968af49d is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37./info:regioninfo/1734308112541/Put/seqid=0
2024-12-16T00:16:12,370 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-16T00:16:12,372 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/273a1c2467264ada9dd23063795252ce to hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/273a1c2467264ada9dd23063795252ce
2024-12-16T00:16:12,372 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/b66def675ade4c1a82fd8945c53a02cf to hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/b66def675ade4c1a82fd8945c53a02cf
2024-12-16T00:16:12,372 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/9850a7633e3e4807969a4346cd275b2b to hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/info/9850a7633e3e4807969a4346cd275b2b
2024-12-16T00:16:12,377 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741849_1025 (size=8430)
2024-12-16T00:16:12,377 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/d57841ff468eaa4432d9a357c67b4a37/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-12-16T00:16:12,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741849_1025 (size=8430)
2024-12-16T00:16:12,379 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.79 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/.tmp/info/a10566e4a1e241ec9b8817d4968af49d
2024-12-16T00:16:12,379 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:16:12,379 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d57841ff468eaa4432d9a357c67b4a37:

2024-12-16T00:16:12,379 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1734308112162.d57841ff468eaa4432d9a357c67b4a37.
2024-12-16T00:16:12,379 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 590e4a408d4f83e289cb88cefe762b5c, disabling compactions & flushes
2024-12-16T00:16:12,379 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:16:12,379 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:16:12,379 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c. after waiting 0 ms
2024-12-16T00:16:12,379 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:16:12,383 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/namespace/590e4a408d4f83e289cb88cefe762b5c/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-16T00:16:12,384 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:16:12,384 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 590e4a408d4f83e289cb88cefe762b5c:

2024-12-16T00:16:12,384 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1734308111506.590e4a408d4f83e289cb88cefe762b5c.
2024-12-16T00:16:12,397 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/.tmp/table/12e0b4b7ff55440b8a062fd988373033 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1734308112548/Put/seqid=0
2024-12-16T00:16:12,401 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741850_1026 (size=5532)
2024-12-16T00:16:12,401 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741850_1026 (size=5532)
2024-12-16T00:16:12,402 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=264 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/.tmp/table/12e0b4b7ff55440b8a062fd988373033
2024-12-16T00:16:12,408 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/.tmp/info/a10566e4a1e241ec9b8817d4968af49d as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/info/a10566e4a1e241ec9b8817d4968af49d
2024-12-16T00:16:12,413 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/info/a10566e4a1e241ec9b8817d4968af49d, entries=20, sequenceid=14, filesize=8.2 K
2024-12-16T00:16:12,414 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/.tmp/table/12e0b4b7ff55440b8a062fd988373033 as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/table/12e0b4b7ff55440b8a062fd988373033
2024-12-16T00:16:12,419 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/table/12e0b4b7ff55440b8a062fd988373033, entries=4, sequenceid=14, filesize=5.4 K
2024-12-16T00:16:12,420 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~3.05 KB/3122, heapSize ~5.27 KB/5400, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=14, compaction requested=false
2024-12-16T00:16:12,425 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1
2024-12-16T00:16:12,425 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-16T00:16:12,425 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-16T00:16:12,426 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:16:12,426 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-16T00:16:12,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:12,547 INFO  [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1250): stopping server b878af803003,36481,1734308110514; all regions closed.
2024-12-16T00:16:12,548 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514
2024-12-16T00:16:12,549 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741834_1010 (size=4570)
2024-12-16T00:16:12,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741834_1010 (size=4570)
2024-12-16T00:16:12,552 DEBUG [RS:0;b878af803003:36481 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/oldWALs
2024-12-16T00:16:12,552 INFO  [RS:0;b878af803003:36481 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog b878af803003%2C36481%2C1734308110514.meta:.meta(num 1734308111405)
2024-12-16T00:16:12,553 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/WALs/b878af803003,36481,1734308110514
2024-12-16T00:16:12,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741847_1023 (size=1545)
2024-12-16T00:16:12,555 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741847_1023 (size=1545)
2024-12-16T00:16:12,559 DEBUG [RS:0;b878af803003:36481 {}] wal.AbstractFSWAL(1071): Moved 2 WAL file(s) to /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/oldWALs
2024-12-16T00:16:12,559 INFO  [RS:0;b878af803003:36481 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog b878af803003%2C36481%2C1734308110514:(num 1734308172321)
2024-12-16T00:16:12,559 DEBUG [RS:0;b878af803003:36481 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:16:12,559 INFO  [RS:0;b878af803003:36481 {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:16:12,559 INFO  [RS:0;b878af803003:36481 {}] hbase.ChoreService(370): Chore service for: regionserver/b878af803003:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-16T00:16:12,560 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:16:12,560 INFO  [RS:0;b878af803003:36481 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36481
2024-12-16T00:16:12,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:16:12,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b878af803003,36481,1734308110514
2024-12-16T00:16:12,579 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b878af803003,36481,1734308110514]
2024-12-16T00:16:12,579 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing b878af803003,36481,1734308110514; numProcessing=1
2024-12-16T00:16:12,587 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/b878af803003,36481,1734308110514 already deleted, retry=false
2024-12-16T00:16:12,587 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; b878af803003,36481,1734308110514 expired; onlineServers=0
2024-12-16T00:16:12,587 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,42851,1734308110350' *****
2024-12-16T00:16:12,587 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-16T00:16:12,588 DEBUG [M:0;b878af803003:42851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d8cf98c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:16:12,588 INFO  [M:0;b878af803003:42851 {}] regionserver.HRegionServer(1224): stopping server b878af803003,42851,1734308110350
2024-12-16T00:16:12,588 INFO  [M:0;b878af803003:42851 {}] regionserver.HRegionServer(1250): stopping server b878af803003,42851,1734308110350; all regions closed.
2024-12-16T00:16:12,588 DEBUG [M:0;b878af803003:42851 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:16:12,588 DEBUG [M:0;b878af803003:42851 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-16T00:16:12,588 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-16T00:16:12,588 DEBUG [M:0;b878af803003:42851 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-16T00:16:12,588 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308110787 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308110787,5,FailOnTimeoutGroup]
2024-12-16T00:16:12,588 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308110787 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308110787,5,FailOnTimeoutGroup]
2024-12-16T00:16:12,589 INFO  [M:0;b878af803003:42851 {}] hbase.ChoreService(370): Chore service for: master/b878af803003:0 had [] on shutdown
2024-12-16T00:16:12,589 DEBUG [M:0;b878af803003:42851 {}] master.HMaster(1733): Stopping service threads
2024-12-16T00:16:12,589 INFO  [M:0;b878af803003:42851 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-16T00:16:12,589 INFO  [M:0;b878af803003:42851 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-16T00:16:12,589 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-16T00:16:12,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-16T00:16:12,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:12,596 DEBUG [M:0;b878af803003:42851 {}] zookeeper.ZKUtil(347): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-16T00:16:12,596 WARN  [M:0;b878af803003:42851 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-16T00:16:12,596 INFO  [M:0;b878af803003:42851 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-16T00:16:12,596 INFO  [M:0;b878af803003:42851 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-16T00:16:12,596 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:16:12,596 DEBUG [M:0;b878af803003:42851 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:16:12,596 INFO  [M:0;b878af803003:42851 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:16:12,596 DEBUG [M:0;b878af803003:42851 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:16:12,597 DEBUG [M:0;b878af803003:42851 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:16:12,597 DEBUG [M:0;b878af803003:42851 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:16:12,597 INFO  [M:0;b878af803003:42851 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=65.09 KB heapSize=81.71 KB
2024-12-16T00:16:12,617 DEBUG [M:0;b878af803003:42851 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/362420b1dd3440d6a7f33b9008db1e18 is 82, key is hbase:meta,,1/info:regioninfo/1734308111426/Put/seqid=0
2024-12-16T00:16:12,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741851_1027 (size=5672)
2024-12-16T00:16:12,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741851_1027 (size=5672)
2024-12-16T00:16:12,626 INFO  [M:0;b878af803003:42851 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/362420b1dd3440d6a7f33b9008db1e18
2024-12-16T00:16:12,644 DEBUG [M:0;b878af803003:42851 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/90034ed27b68476bacdda304905fbfe6 is 799, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1734308112600/Put/seqid=0
2024-12-16T00:16:12,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741852_1028 (size=8357)
2024-12-16T00:16:12,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741852_1028 (size=8357)
2024-12-16T00:16:12,649 INFO  [M:0;b878af803003:42851 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.48 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/90034ed27b68476bacdda304905fbfe6
2024-12-16T00:16:12,653 INFO  [M:0;b878af803003:42851 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 90034ed27b68476bacdda304905fbfe6
2024-12-16T00:16:12,668 DEBUG [M:0;b878af803003:42851 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/db5553d40f4c4898af51073681fff9e4 is 69, key is b878af803003,36481,1734308110514/rs:state/1734308110868/Put/seqid=0
2024-12-16T00:16:12,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741853_1029 (size=5156)
2024-12-16T00:16:12,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741853_1029 (size=5156)
2024-12-16T00:16:12,674 INFO  [M:0;b878af803003:42851 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/db5553d40f4c4898af51073681fff9e4
2024-12-16T00:16:12,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:16:12,679 INFO  [RS:0;b878af803003:36481 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,36481,1734308110514; zookeeper connection closed.
2024-12-16T00:16:12,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36481-0x1002c30bc810001, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:16:12,680 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@46e11c6d {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@46e11c6d
2024-12-16T00:16:12,680 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-16T00:16:12,691 DEBUG [M:0;b878af803003:42851 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f347ccf88a0849d4b47bba947bd2daba is 52, key is load_balancer_on/state:d/1734308112157/Put/seqid=0
2024-12-16T00:16:12,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741854_1030 (size=5056)
2024-12-16T00:16:12,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741854_1030 (size=5056)
2024-12-16T00:16:12,696 INFO  [M:0;b878af803003:42851 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f347ccf88a0849d4b47bba947bd2daba
2024-12-16T00:16:12,701 DEBUG [M:0;b878af803003:42851 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/362420b1dd3440d6a7f33b9008db1e18 as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/362420b1dd3440d6a7f33b9008db1e18
2024-12-16T00:16:12,706 INFO  [M:0;b878af803003:42851 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/362420b1dd3440d6a7f33b9008db1e18, entries=8, sequenceid=184, filesize=5.5 K
2024-12-16T00:16:12,707 DEBUG [M:0;b878af803003:42851 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/90034ed27b68476bacdda304905fbfe6 as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/90034ed27b68476bacdda304905fbfe6
2024-12-16T00:16:12,712 INFO  [M:0;b878af803003:42851 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 90034ed27b68476bacdda304905fbfe6
2024-12-16T00:16:12,712 INFO  [M:0;b878af803003:42851 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/90034ed27b68476bacdda304905fbfe6, entries=21, sequenceid=184, filesize=8.2 K
2024-12-16T00:16:12,713 DEBUG [M:0;b878af803003:42851 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/db5553d40f4c4898af51073681fff9e4 as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/db5553d40f4c4898af51073681fff9e4
2024-12-16T00:16:12,717 INFO  [M:0;b878af803003:42851 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/db5553d40f4c4898af51073681fff9e4, entries=1, sequenceid=184, filesize=5.0 K
2024-12-16T00:16:12,717 DEBUG [M:0;b878af803003:42851 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f347ccf88a0849d4b47bba947bd2daba as hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f347ccf88a0849d4b47bba947bd2daba
2024-12-16T00:16:12,722 INFO  [M:0;b878af803003:42851 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34017/user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f347ccf88a0849d4b47bba947bd2daba, entries=1, sequenceid=184, filesize=4.9 K
2024-12-16T00:16:12,723 INFO  [M:0;b878af803003:42851 {}] regionserver.HRegion(3040): Finished flush of dataSize ~65.09 KB/66649, heapSize ~81.65 KB/83608, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 126ms, sequenceid=184, compaction requested=false
2024-12-16T00:16:12,724 INFO  [M:0;b878af803003:42851 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:16:12,724 DEBUG [M:0;b878af803003:42851 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:16:12,724 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/c212c0de-1e8c-9703-211c-641060846577/MasterData/WALs/b878af803003,42851,1734308110350
2024-12-16T00:16:12,726 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39165 is added to blk_1073741830_1006 (size=79170)
2024-12-16T00:16:12,726 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46609 is added to blk_1073741830_1006 (size=79170)
2024-12-16T00:16:12,726 INFO  [M:0;b878af803003:42851 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-16T00:16:12,726 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:16:12,726 INFO  [M:0;b878af803003:42851 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:42851
2024-12-16T00:16:12,736 DEBUG [M:0;b878af803003:42851 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/b878af803003,42851,1734308110350 already deleted, retry=false
2024-12-16T00:16:12,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:16:12,846 INFO  [M:0;b878af803003:42851 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,42851,1734308110350; zookeeper connection closed.
2024-12-16T00:16:12,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42851-0x1002c30bc810000, quorum=127.0.0.1:53278, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:16:12,852 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@723a5a8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:16:12,852 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4333752{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:16:12,852 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:16:12,853 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3271ec70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:16:12,853 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57736e47{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/hadoop.log.dir/,STOPPED}
2024-12-16T00:16:12,855 WARN  [BP-1194759261-172.17.0.2-1734308108522 heartbeating to localhost/127.0.0.1:34017 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:16:12,855 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:16:12,855 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:16:12,855 WARN  [BP-1194759261-172.17.0.2-1734308108522 heartbeating to localhost/127.0.0.1:34017 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1194759261-172.17.0.2-1734308108522 (Datanode Uuid 459e13cd-ac40-4e4e-ab63-c40244732e37) service to localhost/127.0.0.1:34017
2024-12-16T00:16:12,856 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553/dfs/data/data3/current/BP-1194759261-172.17.0.2-1734308108522 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:16:12,857 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553/dfs/data/data4/current/BP-1194759261-172.17.0.2-1734308108522 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:16:12,857 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:16:12,860 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1242d2b6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:16:12,860 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@56b096cd{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:16:12,861 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:16:12,861 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75942022{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:16:12,861 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4884ca25{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/hadoop.log.dir/,STOPPED}
2024-12-16T00:16:12,863 WARN  [BP-1194759261-172.17.0.2-1734308108522 heartbeating to localhost/127.0.0.1:34017 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:16:12,863 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:16:12,863 WARN  [BP-1194759261-172.17.0.2-1734308108522 heartbeating to localhost/127.0.0.1:34017 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1194759261-172.17.0.2-1734308108522 (Datanode Uuid 78c86dbb-836a-4101-b32f-f044bcda4986) service to localhost/127.0.0.1:34017
2024-12-16T00:16:12,863 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:16:12,863 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553/dfs/data/data1/current/BP-1194759261-172.17.0.2-1734308108522 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:16:12,863 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/cluster_51d20329-5698-8f48-2dda-cc79bffea553/dfs/data/data2/current/BP-1194759261-172.17.0.2-1734308108522 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:16:12,864 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:16:12,870 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@60255f8b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:16:12,870 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e62c0ae{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:16:12,870 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:16:12,871 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1649f99b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:16:12,871 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d004e71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/hadoop.log.dir/,STOPPED}
2024-12-16T00:16:12,876 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-16T00:16:12,894 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-16T00:16:12,896 INFO  [regionserver/b878af803003:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:16:12,900 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=111 (was 99) - Thread LEAK? -, OpenFileDescriptor=466 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=40 (was 54), ProcessCount=11 (was 11), AvailableMemoryMB=11904 (was 12040)
2024-12-16T00:16:12,905 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=111, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=40, ProcessCount=11, AvailableMemoryMB=11904
2024-12-16T00:16:12,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-16T00:16:12,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/hadoop.log.dir so I do NOT create it in target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4
2024-12-16T00:16:12,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/271d68ba-3276-9717-9536-72fc91545fdf/hadoop.tmp.dir so I do NOT create it in target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4
2024-12-16T00:16:12,906 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9, deleteOnExit=true
2024-12-16T00:16:12,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-16T00:16:12,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/test.cache.data in system properties and HBase conf
2024-12-16T00:16:12,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/hadoop.tmp.dir in system properties and HBase conf
2024-12-16T00:16:12,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/hadoop.log.dir in system properties and HBase conf
2024-12-16T00:16:12,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-16T00:16:12,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-16T00:16:12,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-16T00:16:12,907 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-16T00:16:12,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:16:12,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:16:12,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-16T00:16:12,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:16:12,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-16T00:16:12,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-16T00:16:12,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:16:12,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:16:12,908 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-16T00:16:12,908 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/nfs.dump.dir in system properties and HBase conf
2024-12-16T00:16:12,908 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/java.io.tmpdir in system properties and HBase conf
2024-12-16T00:16:12,908 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:16:12,908 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-16T00:16:12,908 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-16T00:16:12,920 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:16:13,151 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:16:13,155 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:16:13,156 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:16:13,156 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:16:13,156 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-16T00:16:13,157 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:16:13,157 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a90e359{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:16:13,158 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@691fae27{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:16:13,245 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@d34012c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/java.io.tmpdir/jetty-localhost-42995-hadoop-hdfs-3_4_1-tests_jar-_-any-1229754491459491794/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:16:13,246 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@68bfa336{HTTP/1.1, (http/1.1)}{localhost:42995}
2024-12-16T00:16:13,246 INFO  [Time-limited test {}] server.Server(415): Started @291910ms
2024-12-16T00:16:13,257 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:16:13,411 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:16:13,413 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:16:13,413 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:16:13,413 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:16:13,414 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:16:13,414 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e5c2e9f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:16:13,414 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b886661{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:16:13,503 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@77ca63ce{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/java.io.tmpdir/jetty-localhost-46351-hadoop-hdfs-3_4_1-tests_jar-_-any-3987303502666647704/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:16:13,504 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@39280d35{HTTP/1.1, (http/1.1)}{localhost:46351}
2024-12-16T00:16:13,504 INFO  [Time-limited test {}] server.Server(415): Started @292168ms
2024-12-16T00:16:13,505 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:16:13,526 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:16:13,528 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:16:13,529 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:16:13,529 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:16:13,529 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:16:13,529 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70f0d727{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:16:13,529 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51ad3217{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:16:13,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:13,618 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34dedf66{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/java.io.tmpdir/jetty-localhost-39137-hadoop-hdfs-3_4_1-tests_jar-_-any-15394190276352644321/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:16:13,619 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@81c66a8{HTTP/1.1, (http/1.1)}{localhost:39137}
2024-12-16T00:16:13,619 INFO  [Time-limited test {}] server.Server(415): Started @292283ms
2024-12-16T00:16:13,619 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:16:14,189 WARN  [Thread-1738 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9/dfs/data/data1/current/BP-518996559-172.17.0.2-1734308172926/current, will proceed with Du for space computation calculation, 
2024-12-16T00:16:14,189 WARN  [Thread-1739 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9/dfs/data/data2/current/BP-518996559-172.17.0.2-1734308172926/current, will proceed with Du for space computation calculation, 
2024-12-16T00:16:14,204 WARN  [Thread-1702 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:16:14,207 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x62813937b0e36853 with lease ID 0x384370df7226f5ce: Processing first storage report for DS-45784aaf-0163-450a-852c-c65b9ce64cdf from datanode DatanodeRegistration(127.0.0.1:39563, datanodeUuid=a4bd9524-63e4-4198-9cef-1a4439074e39, infoPort=43035, infoSecurePort=0, ipcPort=36109, storageInfo=lv=-57;cid=testClusterID;nsid=310483425;c=1734308172926)
2024-12-16T00:16:14,207 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x62813937b0e36853 with lease ID 0x384370df7226f5ce: from storage DS-45784aaf-0163-450a-852c-c65b9ce64cdf node DatanodeRegistration(127.0.0.1:39563, datanodeUuid=a4bd9524-63e4-4198-9cef-1a4439074e39, infoPort=43035, infoSecurePort=0, ipcPort=36109, storageInfo=lv=-57;cid=testClusterID;nsid=310483425;c=1734308172926), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:16:14,207 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x62813937b0e36853 with lease ID 0x384370df7226f5ce: Processing first storage report for DS-1486f5e9-dc53-457f-aa4e-1061b268e6b1 from datanode DatanodeRegistration(127.0.0.1:39563, datanodeUuid=a4bd9524-63e4-4198-9cef-1a4439074e39, infoPort=43035, infoSecurePort=0, ipcPort=36109, storageInfo=lv=-57;cid=testClusterID;nsid=310483425;c=1734308172926)
2024-12-16T00:16:14,207 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x62813937b0e36853 with lease ID 0x384370df7226f5ce: from storage DS-1486f5e9-dc53-457f-aa4e-1061b268e6b1 node DatanodeRegistration(127.0.0.1:39563, datanodeUuid=a4bd9524-63e4-4198-9cef-1a4439074e39, infoPort=43035, infoSecurePort=0, ipcPort=36109, storageInfo=lv=-57;cid=testClusterID;nsid=310483425;c=1734308172926), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:16:14,399 WARN  [Thread-1749 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9/dfs/data/data3/current/BP-518996559-172.17.0.2-1734308172926/current, will proceed with Du for space computation calculation, 
2024-12-16T00:16:14,399 WARN  [Thread-1750 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9/dfs/data/data4/current/BP-518996559-172.17.0.2-1734308172926/current, will proceed with Du for space computation calculation, 
2024-12-16T00:16:14,419 WARN  [Thread-1725 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:16:14,421 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x805af789485c3854 with lease ID 0x384370df7226f5cf: Processing first storage report for DS-441c704c-19af-458f-bd5b-4980c593421a from datanode DatanodeRegistration(127.0.0.1:35593, datanodeUuid=44782388-85e2-4e89-a04e-7cb03c5cff56, infoPort=37683, infoSecurePort=0, ipcPort=43171, storageInfo=lv=-57;cid=testClusterID;nsid=310483425;c=1734308172926)
2024-12-16T00:16:14,421 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x805af789485c3854 with lease ID 0x384370df7226f5cf: from storage DS-441c704c-19af-458f-bd5b-4980c593421a node DatanodeRegistration(127.0.0.1:35593, datanodeUuid=44782388-85e2-4e89-a04e-7cb03c5cff56, infoPort=37683, infoSecurePort=0, ipcPort=43171, storageInfo=lv=-57;cid=testClusterID;nsid=310483425;c=1734308172926), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:16:14,421 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x805af789485c3854 with lease ID 0x384370df7226f5cf: Processing first storage report for DS-c4832b3b-43a1-498c-87e8-26a4c575b663 from datanode DatanodeRegistration(127.0.0.1:35593, datanodeUuid=44782388-85e2-4e89-a04e-7cb03c5cff56, infoPort=37683, infoSecurePort=0, ipcPort=43171, storageInfo=lv=-57;cid=testClusterID;nsid=310483425;c=1734308172926)
2024-12-16T00:16:14,421 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x805af789485c3854 with lease ID 0x384370df7226f5cf: from storage DS-c4832b3b-43a1-498c-87e8-26a4c575b663 node DatanodeRegistration(127.0.0.1:35593, datanodeUuid=44782388-85e2-4e89-a04e-7cb03c5cff56, infoPort=37683, infoSecurePort=0, ipcPort=43171, storageInfo=lv=-57;cid=testClusterID;nsid=310483425;c=1734308172926), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:16:14,444 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4
2024-12-16T00:16:14,447 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9/zookeeper_0, clientPort=51475, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-16T00:16:14,448 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=51475
2024-12-16T00:16:14,448 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:16:14,449 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:16:14,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:16:14,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:16:14,457 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e with version=8
2024-12-16T00:16:14,457 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/hbase-staging
2024-12-16T00:16:14,458 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/b878af803003:0 server-side Connection retries=45
2024-12-16T00:16:14,458 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:16:14,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:16:14,459 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:16:14,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:16:14,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:16:14,459 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:16:14,459 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:16:14,459 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:44525
2024-12-16T00:16:14,460 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:16:14,461 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:16:14,462 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:44525 connecting to ZooKeeper ensemble=127.0.0.1:51475
2024-12-16T00:16:14,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:445250x0, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:16:14,520 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44525-0x1002c31b6ee0000 connected
2024-12-16T00:16:14,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:14,588 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:16:14,589 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:16:14,589 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:16:14,590 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44525
2024-12-16T00:16:14,590 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44525
2024-12-16T00:16:14,590 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44525
2024-12-16T00:16:14,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44525
2024-12-16T00:16:14,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44525
2024-12-16T00:16:14,591 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e, hbase.cluster.distributed=false
2024-12-16T00:16:14,609 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/b878af803003:0 server-side Connection retries=45
2024-12-16T00:16:14,609 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:16:14,609 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:16:14,609 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:16:14,609 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:16:14,609 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:16:14,609 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:16:14,609 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:16:14,609 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:42285
2024-12-16T00:16:14,610 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-16T00:16:14,610 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-16T00:16:14,611 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:16:14,612 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:16:14,614 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:42285 connecting to ZooKeeper ensemble=127.0.0.1:51475
2024-12-16T00:16:14,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422850x0, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:16:14,621 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42285-0x1002c31b6ee0001 connected
2024-12-16T00:16:14,621 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:16:14,622 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:16:14,623 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:16:14,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42285
2024-12-16T00:16:14,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42285
2024-12-16T00:16:14,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42285
2024-12-16T00:16:14,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42285
2024-12-16T00:16:14,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42285
2024-12-16T00:16:14,625 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/b878af803003,44525,1734308174458
2024-12-16T00:16:14,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:16:14,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:16:14,629 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b878af803003,44525,1734308174458
2024-12-16T00:16:14,637 DEBUG [M:0;b878af803003:44525 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b878af803003:44525
2024-12-16T00:16:14,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:16:14,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:16:14,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:14,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:14,638 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:16:14,638 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b878af803003,44525,1734308174458 from backup master directory
2024-12-16T00:16:14,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b878af803003,44525,1734308174458
2024-12-16T00:16:14,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:16:14,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:16:14,646 WARN  [master/b878af803003:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:16:14,646 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:16:14,646 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b878af803003,44525,1734308174458
2024-12-16T00:16:14,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:16:14,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:16:14,655 DEBUG [master/b878af803003:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/hbase.id with ID: ac467a50-61fd-40ad-bbeb-d409dfaa59b4
2024-12-16T00:16:14,665 INFO  [master/b878af803003:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:16:14,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:14,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:14,676 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:16:14,676 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:16:14,677 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-16T00:16:14,677 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-16T00:16:14,678 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:16:14,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:16:14,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:16:14,686 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store
2024-12-16T00:16:14,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:16:14,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:16:14,695 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:16:14,695 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:16:14,695 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:16:14,695 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:16:14,695 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:16:14,695 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:16:14,695 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:16:14,695 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:16:14,696 WARN  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/.initializing
2024-12-16T00:16:14,696 DEBUG [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/WALs/b878af803003,44525,1734308174458
2024-12-16T00:16:14,700 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C44525%2C1734308174458, suffix=, logDir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/WALs/b878af803003,44525,1734308174458, archiveDir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/oldWALs, maxLogs=10
2024-12-16T00:16:14,700 INFO  [master/b878af803003:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C44525%2C1734308174458.1734308174700
2024-12-16T00:16:14,707 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/WALs/b878af803003,44525,1734308174458/b878af803003%2C44525%2C1734308174458.1734308174700
2024-12-16T00:16:14,707 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37683:37683),(127.0.0.1/127.0.0.1:43035:43035)]
2024-12-16T00:16:14,707 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:16:14,707 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:16:14,708 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:16:14,708 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:16:14,709 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:16:14,711 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-16T00:16:14,711 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:14,712 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:16:14,712 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:16:14,713 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-16T00:16:14,713 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:14,714 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:16:14,714 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:16:14,715 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-16T00:16:14,715 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:14,715 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:16:14,716 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:16:14,717 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-16T00:16:14,717 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:14,717 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:16:14,718 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:16:14,719 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:16:14,721 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-16T00:16:14,722 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:16:14,724 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:16:14,725 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=793655, jitterRate=0.009185418486595154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-16T00:16:14,725 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:16:14,726 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-16T00:16:14,729 DEBUG [master/b878af803003:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fcb8124, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:16:14,729 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-16T00:16:14,729 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-16T00:16:14,729 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-16T00:16:14,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-16T00:16:14,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-16T00:16:14,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-16T00:16:14,730 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-16T00:16:14,732 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-16T00:16:14,732 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-16T00:16:14,737 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-16T00:16:14,737 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-16T00:16:14,738 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-16T00:16:14,746 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-16T00:16:14,746 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-16T00:16:14,747 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-16T00:16:14,754 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-16T00:16:14,754 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-16T00:16:14,762 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-16T00:16:14,764 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-16T00:16:14,771 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-16T00:16:14,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:16:14,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:16:14,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:14,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:14,780 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=b878af803003,44525,1734308174458, sessionid=0x1002c31b6ee0000, setting cluster-up flag (Was=false)
2024-12-16T00:16:14,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:14,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:14,821 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-16T00:16:14,823 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,44525,1734308174458
2024-12-16T00:16:14,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:14,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:14,863 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-16T00:16:14,864 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,44525,1734308174458
2024-12-16T00:16:14,866 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-16T00:16:14,866 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-16T00:16:14,866 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-16T00:16:14,867 DEBUG [master/b878af803003:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b878af803003,44525,1734308174458
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-16T00:16:14,867 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:16:14,867 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:16:14,867 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:16:14,867 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:16:14,867 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b878af803003:0, corePoolSize=10, maxPoolSize=10
2024-12-16T00:16:14,867 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,867 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:16:14,867 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,869 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734308204869
2024-12-16T00:16:14,869 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-16T00:16:14,869 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-16T00:16:14,869 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-16T00:16:14,869 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-16T00:16:14,869 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-16T00:16:14,869 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-16T00:16:14,869 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:16:14,869 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-16T00:16:14,869 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:14,870 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-16T00:16:14,870 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-16T00:16:14,870 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-16T00:16:14,870 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-16T00:16:14,870 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-16T00:16:14,870 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:14,871 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:16:14,871 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308174870,5,FailOnTimeoutGroup]
2024-12-16T00:16:14,871 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308174871,5,FailOnTimeoutGroup]
2024-12-16T00:16:14,871 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:14,871 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-16T00:16:14,871 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:14,872 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:14,876 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:16:14,876 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:16:14,877 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-16T00:16:14,877 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e
2024-12-16T00:16:14,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:16:14,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:16:14,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:16:14,885 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:16:14,886 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:16:14,886 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:14,886 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:16:14,886 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:16:14,887 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:16:14,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:14,888 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:16:14,888 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:16:14,889 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:16:14,889 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:14,889 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:16:14,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740
2024-12-16T00:16:14,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740
2024-12-16T00:16:14,891 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:16:14,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:16:14,895 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:16:14,895 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=751442, jitterRate=-0.04449322819709778}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:16:14,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:16:14,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:16:14,895 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:16:14,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:16:14,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:16:14,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:16:14,896 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-16T00:16:14,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:16:14,896 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:16:14,896 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-16T00:16:14,897 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-16T00:16:14,897 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-16T00:16:14,898 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-16T00:16:14,934 DEBUG [RS:0;b878af803003:42285 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b878af803003:42285
2024-12-16T00:16:14,935 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1008): ClusterId : ac467a50-61fd-40ad-bbeb-d409dfaa59b4
2024-12-16T00:16:14,935 DEBUG [RS:0;b878af803003:42285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-16T00:16:14,945 DEBUG [RS:0;b878af803003:42285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-16T00:16:14,945 DEBUG [RS:0;b878af803003:42285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-16T00:16:14,955 DEBUG [RS:0;b878af803003:42285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-16T00:16:14,955 DEBUG [RS:0;b878af803003:42285 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36d43419, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:16:14,956 DEBUG [RS:0;b878af803003:42285 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708b5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:16:14,956 INFO  [RS:0;b878af803003:42285 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-16T00:16:14,956 INFO  [RS:0;b878af803003:42285 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-16T00:16:14,956 DEBUG [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-16T00:16:14,957 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(3073): reportForDuty to master=b878af803003,44525,1734308174458 with isa=b878af803003/172.17.0.2:42285, startcode=1734308174608
2024-12-16T00:16:14,957 DEBUG [RS:0;b878af803003:42285 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:16:14,960 INFO  [RS-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43435, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:16:14,960 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44525 {}] master.ServerManager(332): Checking decommissioned status of RegionServer b878af803003,42285,1734308174608
2024-12-16T00:16:14,960 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44525 {}] master.ServerManager(486): Registering regionserver=b878af803003,42285,1734308174608
2024-12-16T00:16:14,962 DEBUG [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e
2024-12-16T00:16:14,962 DEBUG [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:38027
2024-12-16T00:16:14,962 DEBUG [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-16T00:16:14,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:16:14,971 DEBUG [RS:0;b878af803003:42285 {}] zookeeper.ZKUtil(111): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b878af803003,42285,1734308174608
2024-12-16T00:16:14,971 WARN  [RS:0;b878af803003:42285 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:16:14,971 INFO  [RS:0;b878af803003:42285 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:16:14,972 DEBUG [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608
2024-12-16T00:16:14,972 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b878af803003,42285,1734308174608]
2024-12-16T00:16:14,977 DEBUG [RS:0;b878af803003:42285 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-16T00:16:14,978 INFO  [RS:0;b878af803003:42285 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-16T00:16:14,980 INFO  [RS:0;b878af803003:42285 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-16T00:16:14,981 INFO  [RS:0;b878af803003:42285 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-16T00:16:14,981 INFO  [RS:0;b878af803003:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:14,981 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-16T00:16:14,982 INFO  [RS:0;b878af803003:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:16:14,983 DEBUG [RS:0;b878af803003:42285 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:16:14,984 INFO  [RS:0;b878af803003:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:14,984 INFO  [RS:0;b878af803003:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:14,984 INFO  [RS:0;b878af803003:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:14,984 INFO  [RS:0;b878af803003:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:14,984 INFO  [RS:0;b878af803003:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42285,1734308174608-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:16:14,999 INFO  [RS:0;b878af803003:42285 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-16T00:16:14,999 INFO  [RS:0;b878af803003:42285 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42285,1734308174608-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:15,010 INFO  [RS:0;b878af803003:42285 {}] regionserver.Replication(204): b878af803003,42285,1734308174608 started
2024-12-16T00:16:15,010 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1767): Serving as b878af803003,42285,1734308174608, RpcServer on b878af803003/172.17.0.2:42285, sessionid=0x1002c31b6ee0001
2024-12-16T00:16:15,010 DEBUG [RS:0;b878af803003:42285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-16T00:16:15,010 DEBUG [RS:0;b878af803003:42285 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b878af803003,42285,1734308174608
2024-12-16T00:16:15,010 DEBUG [RS:0;b878af803003:42285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,42285,1734308174608'
2024-12-16T00:16:15,010 DEBUG [RS:0;b878af803003:42285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-16T00:16:15,011 DEBUG [RS:0;b878af803003:42285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-16T00:16:15,011 DEBUG [RS:0;b878af803003:42285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-16T00:16:15,011 DEBUG [RS:0;b878af803003:42285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-16T00:16:15,011 DEBUG [RS:0;b878af803003:42285 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b878af803003,42285,1734308174608
2024-12-16T00:16:15,011 DEBUG [RS:0;b878af803003:42285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,42285,1734308174608'
2024-12-16T00:16:15,011 DEBUG [RS:0;b878af803003:42285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-16T00:16:15,012 DEBUG [RS:0;b878af803003:42285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-16T00:16:15,012 DEBUG [RS:0;b878af803003:42285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-16T00:16:15,012 INFO  [RS:0;b878af803003:42285 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-16T00:16:15,012 INFO  [RS:0;b878af803003:42285 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-16T00:16:15,048 WARN  [b878af803003:44525 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-16T00:16:15,116 INFO  [RS:0;b878af803003:42285 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C42285%2C1734308174608, suffix=, logDir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608, archiveDir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/oldWALs, maxLogs=32
2024-12-16T00:16:15,117 INFO  [RS:0;b878af803003:42285 {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C42285%2C1734308174608.1734308175117
2024-12-16T00:16:15,125 INFO  [RS:0;b878af803003:42285 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608/b878af803003%2C42285%2C1734308174608.1734308175117
2024-12-16T00:16:15,125 DEBUG [RS:0;b878af803003:42285 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43035:43035),(127.0.0.1/127.0.0.1:37683:37683)]
2024-12-16T00:16:15,299 DEBUG [b878af803003:44525 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-16T00:16:15,300 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b878af803003,42285,1734308174608
2024-12-16T00:16:15,302 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,42285,1734308174608, state=OPENING
2024-12-16T00:16:15,313 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-16T00:16:15,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:15,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:15,323 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=b878af803003,42285,1734308174608}]
2024-12-16T00:16:15,323 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:16:15,323 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:16:15,478 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to b878af803003,42285,1734308174608
2024-12-16T00:16:15,479 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-16T00:16:15,483 INFO  [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47366, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-16T00:16:15,489 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-16T00:16:15,489 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:16:15,491 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C42285%2C1734308174608.meta, suffix=.meta, logDir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608, archiveDir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/oldWALs, maxLogs=32
2024-12-16T00:16:15,492 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C42285%2C1734308174608.meta.1734308175492.meta
2024-12-16T00:16:15,499 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608/b878af803003%2C42285%2C1734308174608.meta.1734308175492.meta
2024-12-16T00:16:15,499 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37683:37683),(127.0.0.1/127.0.0.1:43035:43035)]
2024-12-16T00:16:15,499 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:16:15,500 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-16T00:16:15,500 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-16T00:16:15,500 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-16T00:16:15,500 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-16T00:16:15,500 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:16:15,500 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-16T00:16:15,500 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-16T00:16:15,502 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:16:15,502 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:16:15,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:15,503 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:16:15,503 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:16:15,504 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:16:15,504 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:15,504 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:16:15,504 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:16:15,505 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:16:15,505 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:15,506 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:16:15,507 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740
2024-12-16T00:16:15,508 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740
2024-12-16T00:16:15,509 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:16:15,511 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:16:15,511 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=866521, jitterRate=0.10183879733085632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:16:15,512 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:16:15,512 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734308175478
2024-12-16T00:16:15,514 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-16T00:16:15,514 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-16T00:16:15,514 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,42285,1734308174608
2024-12-16T00:16:15,515 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,42285,1734308174608, state=OPEN
2024-12-16T00:16:15,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:15,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:16:15,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:16:15,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:16:15,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:16:15,566 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-16T00:16:15,566 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=b878af803003,42285,1734308174608 in 241 msec
2024-12-16T00:16:15,567 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-16T00:16:15,567 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 669 msec
2024-12-16T00:16:15,568 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 702 msec
2024-12-16T00:16:15,568 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734308175568, completionTime=-1
2024-12-16T00:16:15,568 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-16T00:16:15,569 DEBUG [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-16T00:16:15,569 DEBUG [hconnection-0x192f029-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:16:15,570 INFO  [RS-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47374, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:16:15,571 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-16T00:16:15,571 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734308235571
2024-12-16T00:16:15,571 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734308295571
2024-12-16T00:16:15,571 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec
2024-12-16T00:16:15,596 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44525,1734308174458-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:15,596 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44525,1734308174458-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:15,596 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44525,1734308174458-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:15,596 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b878af803003:44525, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:15,596 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:15,597 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-16T00:16:15,597 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:16:15,598 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-16T00:16:15,598 DEBUG [master/b878af803003:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-16T00:16:15,599 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-16T00:16:15,600 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:15,601 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-16T00:16:15,609 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:16:15,610 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:16:15,612 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 5982437be6e272746f8b60f107f87ef4, NAME => 'hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e
2024-12-16T00:16:15,620 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:16:15,620 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:16:15,621 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:16:15,621 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 5982437be6e272746f8b60f107f87ef4, disabling compactions & flushes
2024-12-16T00:16:15,621 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:16:15,621 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:16:15,621 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4. after waiting 0 ms
2024-12-16T00:16:15,621 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:16:15,621 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:16:15,621 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 5982437be6e272746f8b60f107f87ef4:

2024-12-16T00:16:15,622 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-16T00:16:15,622 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734308175622"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734308175622"}]},"ts":"1734308175622"}
2024-12-16T00:16:15,624 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-16T00:16:15,624 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-16T00:16:15,625 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308175625"}]},"ts":"1734308175625"}
2024-12-16T00:16:15,626 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-16T00:16:15,646 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=5982437be6e272746f8b60f107f87ef4, ASSIGN}]
2024-12-16T00:16:15,647 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=5982437be6e272746f8b60f107f87ef4, ASSIGN
2024-12-16T00:16:15,648 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=5982437be6e272746f8b60f107f87ef4, ASSIGN; state=OFFLINE, location=b878af803003,42285,1734308174608; forceNewPlan=false, retain=false
2024-12-16T00:16:15,799 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=5982437be6e272746f8b60f107f87ef4, regionState=OPENING, regionLocation=b878af803003,42285,1734308174608
2024-12-16T00:16:15,804 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 5982437be6e272746f8b60f107f87ef4, server=b878af803003,42285,1734308174608}]
2024-12-16T00:16:15,960 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to b878af803003,42285,1734308174608
2024-12-16T00:16:15,969 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:16:15,969 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 5982437be6e272746f8b60f107f87ef4, NAME => 'hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:16:15,970 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 5982437be6e272746f8b60f107f87ef4
2024-12-16T00:16:15,970 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:16:15,970 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 5982437be6e272746f8b60f107f87ef4
2024-12-16T00:16:15,970 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 5982437be6e272746f8b60f107f87ef4
2024-12-16T00:16:15,972 INFO  [StoreOpener-5982437be6e272746f8b60f107f87ef4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5982437be6e272746f8b60f107f87ef4 
2024-12-16T00:16:15,974 INFO  [StoreOpener-5982437be6e272746f8b60f107f87ef4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5982437be6e272746f8b60f107f87ef4 columnFamilyName info
2024-12-16T00:16:15,974 DEBUG [StoreOpener-5982437be6e272746f8b60f107f87ef4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:15,975 INFO  [StoreOpener-5982437be6e272746f8b60f107f87ef4-1 {}] regionserver.HStore(327): Store=5982437be6e272746f8b60f107f87ef4/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:16:15,976 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/namespace/5982437be6e272746f8b60f107f87ef4
2024-12-16T00:16:15,976 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/namespace/5982437be6e272746f8b60f107f87ef4
2024-12-16T00:16:15,979 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 5982437be6e272746f8b60f107f87ef4
2024-12-16T00:16:15,982 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/namespace/5982437be6e272746f8b60f107f87ef4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:16:15,982 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 5982437be6e272746f8b60f107f87ef4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=873909, jitterRate=0.11123305559158325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:16:15,982 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 5982437be6e272746f8b60f107f87ef4:

2024-12-16T00:16:15,983 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4., pid=6, masterSystemTime=1734308175960
2024-12-16T00:16:15,985 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:16:15,985 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:16:15,986 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=5982437be6e272746f8b60f107f87ef4, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,42285,1734308174608
2024-12-16T00:16:15,990 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-16T00:16:15,990 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 5982437be6e272746f8b60f107f87ef4, server=b878af803003,42285,1734308174608 in 184 msec
2024-12-16T00:16:15,992 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-16T00:16:15,992 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=5982437be6e272746f8b60f107f87ef4, ASSIGN in 344 msec
2024-12-16T00:16:15,992 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-16T00:16:15,993 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308175992"}]},"ts":"1734308175992"}
2024-12-16T00:16:15,994 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-16T00:16:16,030 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-16T00:16:16,031 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-16T00:16:16,033 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 434 msec
2024-12-16T00:16:16,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:16:16,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:16,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:16:16,042 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-16T00:16:16,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:16:16,065 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 22 msec
2024-12-16T00:16:16,075 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-16T00:16:16,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:16:16,098 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 23 msec
2024-12-16T00:16:16,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-16T00:16:16,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-16T00:16:16,138 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.491sec
2024-12-16T00:16:16,138 INFO  [master/b878af803003:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-16T00:16:16,138 INFO  [master/b878af803003:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-16T00:16:16,138 INFO  [master/b878af803003:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-16T00:16:16,138 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-16T00:16:16,138 INFO  [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-16T00:16:16,138 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44525,1734308174458-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:16:16,138 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44525,1734308174458-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-16T00:16:16,141 DEBUG [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-16T00:16:16,141 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-16T00:16:16,141 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,44525,1734308174458-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:16:16,230 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x13783cfc to 127.0.0.1:51475 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6d3a8e3
2024-12-16T00:16:16,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dbda167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:16:16,241 DEBUG [hconnection-0x5b10ecdc-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:16:16,242 INFO  [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47384, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:16:16,244 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=b878af803003,44525,1734308174458
2024-12-16T00:16:16,244 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:16:16,248 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-16T00:16:16,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-16T00:16:16,251 INFO  [RS-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36764, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-16T00:16:16,253 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44525 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-16T00:16:16,253 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44525 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-16T00:16:16,253 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44525 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-16T00:16:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44525 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-12-16T00:16:16,256 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-16T00:16:16,256 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:16,256 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44525 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 9
2024-12-16T00:16:16,257 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-16T00:16:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44525 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-16T00:16:16,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741837_1013 (size=381)
2024-12-16T00:16:16,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741837_1013 (size=381)
2024-12-16T00:16:16,265 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f4d6c0aff037d3eb34d03175b6108e70, NAME => 'TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e
2024-12-16T00:16:16,270 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741838_1014 (size=64)
2024-12-16T00:16:16,270 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741838_1014 (size=64)
2024-12-16T00:16:16,271 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:16:16,271 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing f4d6c0aff037d3eb34d03175b6108e70, disabling compactions & flushes
2024-12-16T00:16:16,271 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:16,271 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:16,271 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70. after waiting 0 ms
2024-12-16T00:16:16,271 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:16,271 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:16,271 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for f4d6c0aff037d3eb34d03175b6108e70:

2024-12-16T00:16:16,272 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-16T00:16:16,272 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1734308176272"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734308176272"}]},"ts":"1734308176272"}
2024-12-16T00:16:16,273 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-16T00:16:16,274 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-16T00:16:16,274 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308176274"}]},"ts":"1734308176274"}
2024-12-16T00:16:16,276 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-12-16T00:16:16,296 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f4d6c0aff037d3eb34d03175b6108e70, ASSIGN}]
2024-12-16T00:16:16,297 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f4d6c0aff037d3eb34d03175b6108e70, ASSIGN
2024-12-16T00:16:16,298 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f4d6c0aff037d3eb34d03175b6108e70, ASSIGN; state=OFFLINE, location=b878af803003,42285,1734308174608; forceNewPlan=false, retain=false
2024-12-16T00:16:16,449 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=f4d6c0aff037d3eb34d03175b6108e70, regionState=OPENING, regionLocation=b878af803003,42285,1734308174608
2024-12-16T00:16:16,453 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure f4d6c0aff037d3eb34d03175b6108e70, server=b878af803003,42285,1734308174608}]
2024-12-16T00:16:16,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:16,607 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to b878af803003,42285,1734308174608
2024-12-16T00:16:16,616 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:16,616 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => f4d6c0aff037d3eb34d03175b6108e70, NAME => 'TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:16:16,617 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:16,617 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:16:16,617 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:16,617 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:16,619 INFO  [StoreOpener-f4d6c0aff037d3eb34d03175b6108e70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f4d6c0aff037d3eb34d03175b6108e70 
2024-12-16T00:16:16,621 INFO  [StoreOpener-f4d6c0aff037d3eb34d03175b6108e70-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4d6c0aff037d3eb34d03175b6108e70 columnFamilyName info
2024-12-16T00:16:16,621 DEBUG [StoreOpener-f4d6c0aff037d3eb34d03175b6108e70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:16,622 INFO  [StoreOpener-f4d6c0aff037d3eb34d03175b6108e70-1 {}] regionserver.HStore(327): Store=f4d6c0aff037d3eb34d03175b6108e70/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:16:16,623 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:16,623 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:16,626 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:16,629 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:16:16,629 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened f4d6c0aff037d3eb34d03175b6108e70; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=751025, jitterRate=-0.045022547245025635}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:16:16,630 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for f4d6c0aff037d3eb34d03175b6108e70:

2024-12-16T00:16:16,631 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70., pid=11, masterSystemTime=1734308176607
2024-12-16T00:16:16,632 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:16,632 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:16,633 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=f4d6c0aff037d3eb34d03175b6108e70, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,42285,1734308174608
2024-12-16T00:16:16,637 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-16T00:16:16,637 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure f4d6c0aff037d3eb34d03175b6108e70, server=b878af803003,42285,1734308174608 in 182 msec
2024-12-16T00:16:16,639 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-16T00:16:16,639 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f4d6c0aff037d3eb34d03175b6108e70, ASSIGN in 341 msec
2024-12-16T00:16:16,640 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-16T00:16:16,641 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308176640"}]},"ts":"1734308176640"}
2024-12-16T00:16:16,642 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-12-16T00:16:16,681 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-16T00:16:16,685 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRolling in 428 msec
2024-12-16T00:16:16,862 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-16T00:16:16,862 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-16T00:16:16,863 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-16T00:16:17,380 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,381 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,381 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,382 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,382 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,383 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,398 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,398 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,398 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,399 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,399 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,399 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,401 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,401 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,401 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,403 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:17,907 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-16T00:16:17,910 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,911 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,911 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,912 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,912 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,931 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,931 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,933 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,933 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,933 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:17,935 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:18,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:19,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:20,544 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:20,978 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-16T00:16:20,979 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-16T00:16:20,981 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-16T00:16:21,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:22,546 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:23,547 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:24,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:25,550 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44525 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-16T00:16:26,260 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling, procId: 9 completed
2024-12-16T00:16:26,266 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRolling
2024-12-16T00:16:26,266 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:26,281 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing f4d6c0aff037d3eb34d03175b6108e70 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-16T00:16:26,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/5b832ded383c4a5ca2b0e60e372ebe36 is 1080, key is row0001/info:/1734308186271/Put/seqid=0
2024-12-16T00:16:26,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741839_1015 (size=12509)
2024-12-16T00:16:26,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741839_1015 (size=12509)
2024-12-16T00:16:26,305 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/5b832ded383c4a5ca2b0e60e372ebe36
2024-12-16T00:16:26,308 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f4d6c0aff037d3eb34d03175b6108e70, server=b878af803003,42285,1734308174608
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-16T00:16:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:47384 deadline: 1734308196307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f4d6c0aff037d3eb34d03175b6108e70, server=b878af803003,42285,1734308174608
2024-12-16T00:16:26,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/5b832ded383c4a5ca2b0e60e372ebe36 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/5b832ded383c4a5ca2b0e60e372ebe36
2024-12-16T00:16:26,316 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/5b832ded383c4a5ca2b0e60e372ebe36, entries=7, sequenceid=11, filesize=12.2 K
2024-12-16T00:16:26,317 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for f4d6c0aff037d3eb34d03175b6108e70 in 36ms, sequenceid=11, compaction requested=false
2024-12-16T00:16:26,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for f4d6c0aff037d3eb34d03175b6108e70:

2024-12-16T00:16:26,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:27,299 DEBUG [master/b878af803003:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-16T00:16:27,299 DEBUG [master/b878af803003:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=1, created chunk count=12, reused chunk count=39, reuseRatio=76.47%
2024-12-16T00:16:27,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:28,554 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:29,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:30,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:31,557 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:32,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:33,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:33,872 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,873 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,873 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,875 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,893 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,894 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,894 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,894 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,894 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,894 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,896 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,896 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,896 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:33,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,404 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-16T00:16:34,405 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,406 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,406 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,406 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,407 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,407 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,427 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,427 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,427 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,427 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,427 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,428 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,430 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,430 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,430 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,432 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:34,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:35,561 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:36,386 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing f4d6c0aff037d3eb34d03175b6108e70 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-16T00:16:36,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/2be16d0b88d843e39abc7d29907bd162 is 1080, key is row0008/info:/1734308186281/Put/seqid=0
2024-12-16T00:16:36,400 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741840_1016 (size=29761)
2024-12-16T00:16:36,400 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741840_1016 (size=29761)
2024-12-16T00:16:36,401 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/2be16d0b88d843e39abc7d29907bd162
2024-12-16T00:16:36,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/2be16d0b88d843e39abc7d29907bd162 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/2be16d0b88d843e39abc7d29907bd162
2024-12-16T00:16:36,412 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/2be16d0b88d843e39abc7d29907bd162, entries=23, sequenceid=37, filesize=29.1 K
2024-12-16T00:16:36,412 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for f4d6c0aff037d3eb34d03175b6108e70 in 27ms, sequenceid=37, compaction requested=false
2024-12-16T00:16:36,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for f4d6c0aff037d3eb34d03175b6108e70:

2024-12-16T00:16:36,413 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=41.3 K, sizeToCheck=16.0 K
2024-12-16T00:16:36,413 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:16:36,413 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/2be16d0b88d843e39abc7d29907bd162 because midkey is the same as first or last row
2024-12-16T00:16:36,561 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:37,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:38,397 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing f4d6c0aff037d3eb34d03175b6108e70 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-16T00:16:38,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/604c62ce05794f868b3f74a47e3770cb is 1080, key is row0031/info:/1734308196387/Put/seqid=0
2024-12-16T00:16:38,406 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741841_1017 (size=12509)
2024-12-16T00:16:38,406 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741841_1017 (size=12509)
2024-12-16T00:16:38,406 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/604c62ce05794f868b3f74a47e3770cb
2024-12-16T00:16:38,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/604c62ce05794f868b3f74a47e3770cb as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/604c62ce05794f868b3f74a47e3770cb
2024-12-16T00:16:38,416 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/604c62ce05794f868b3f74a47e3770cb, entries=7, sequenceid=47, filesize=12.2 K
2024-12-16T00:16:38,417 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for f4d6c0aff037d3eb34d03175b6108e70 in 20ms, sequenceid=47, compaction requested=true
2024-12-16T00:16:38,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for f4d6c0aff037d3eb34d03175b6108e70:

2024-12-16T00:16:38,417 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=53.5 K, sizeToCheck=16.0 K
2024-12-16T00:16:38,417 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:16:38,417 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/2be16d0b88d843e39abc7d29907bd162 because midkey is the same as first or last row
2024-12-16T00:16:38,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4d6c0aff037d3eb34d03175b6108e70:info, priority=-2147483648, current under compaction store size is 1
2024-12-16T00:16:38,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:16:38,418 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-16T00:16:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:38,419 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing f4d6c0aff037d3eb34d03175b6108e70 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB
2024-12-16T00:16:38,419 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-16T00:16:38,419 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1540): f4d6c0aff037d3eb34d03175b6108e70/info is initiating minor compaction (all files)
2024-12-16T00:16:38,419 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of f4d6c0aff037d3eb34d03175b6108e70/info in TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:38,419 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/5b832ded383c4a5ca2b0e60e372ebe36, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/2be16d0b88d843e39abc7d29907bd162, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/604c62ce05794f868b3f74a47e3770cb] into tmpdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp, totalSize=53.5 K
2024-12-16T00:16:38,420 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5b832ded383c4a5ca2b0e60e372ebe36, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1734308186271
2024-12-16T00:16:38,420 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2be16d0b88d843e39abc7d29907bd162, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1734308186281
2024-12-16T00:16:38,420 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 604c62ce05794f868b3f74a47e3770cb, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1734308196387
2024-12-16T00:16:38,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/d09019b90dd1406aa38b63962ddaa8aa is 1080, key is row0038/info:/1734308198398/Put/seqid=0
2024-12-16T00:16:38,433 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741842_1018 (size=23299)
2024-12-16T00:16:38,433 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741842_1018 (size=23299)
2024-12-16T00:16:38,433 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/d09019b90dd1406aa38b63962ddaa8aa
2024-12-16T00:16:38,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/d09019b90dd1406aa38b63962ddaa8aa as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/d09019b90dd1406aa38b63962ddaa8aa
2024-12-16T00:16:38,444 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4d6c0aff037d3eb34d03175b6108e70#info#compaction#45 average throughput is 12.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:16:38,444 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/b7fe472d429c45b09d5e9401ec54147a is 1080, key is row0001/info:/1734308186271/Put/seqid=0
2024-12-16T00:16:38,445 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/d09019b90dd1406aa38b63962ddaa8aa, entries=17, sequenceid=67, filesize=22.8 K
2024-12-16T00:16:38,446 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=10.51 KB/10760 for f4d6c0aff037d3eb34d03175b6108e70 in 27ms, sequenceid=67, compaction requested=false
2024-12-16T00:16:38,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for f4d6c0aff037d3eb34d03175b6108e70:

2024-12-16T00:16:38,446 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=76.2 K, sizeToCheck=16.0 K
2024-12-16T00:16:38,446 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:16:38,446 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/2be16d0b88d843e39abc7d29907bd162 because midkey is the same as first or last row
2024-12-16T00:16:38,452 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741843_1019 (size=44978)
2024-12-16T00:16:38,452 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741843_1019 (size=44978)
2024-12-16T00:16:38,458 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/b7fe472d429c45b09d5e9401ec54147a as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/b7fe472d429c45b09d5e9401ec54147a
2024-12-16T00:16:38,464 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in f4d6c0aff037d3eb34d03175b6108e70/info of f4d6c0aff037d3eb34d03175b6108e70 into b7fe472d429c45b09d5e9401ec54147a(size=43.9 K), total size for store is 66.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-16T00:16:38,464 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for f4d6c0aff037d3eb34d03175b6108e70:

2024-12-16T00:16:38,464 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70., storeName=f4d6c0aff037d3eb34d03175b6108e70/info, priority=13, startTime=1734308198418; duration=0sec
2024-12-16T00:16:38,464 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=66.7 K, sizeToCheck=16.0 K
2024-12-16T00:16:38,464 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:16:38,465 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/b7fe472d429c45b09d5e9401ec54147a because midkey is the same as first or last row
2024-12-16T00:16:38,465 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:16:38,465 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4d6c0aff037d3eb34d03175b6108e70:info
2024-12-16T00:16:38,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:39,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,446 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing f4d6c0aff037d3eb34d03175b6108e70 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-16T00:16:40,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/a19e4c58dc98405e915763a7cadcd65e is 1080, key is row0055/info:/1734308198419/Put/seqid=0
2024-12-16T00:16:40,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741844_1020 (size=16817)
2024-12-16T00:16:40,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741844_1020 (size=16817)
2024-12-16T00:16:40,457 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/a19e4c58dc98405e915763a7cadcd65e
2024-12-16T00:16:40,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/a19e4c58dc98405e915763a7cadcd65e as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/a19e4c58dc98405e915763a7cadcd65e
2024-12-16T00:16:40,470 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/a19e4c58dc98405e915763a7cadcd65e, entries=11, sequenceid=82, filesize=16.4 K
2024-12-16T00:16:40,471 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=17.86 KB/18292 for f4d6c0aff037d3eb34d03175b6108e70 in 25ms, sequenceid=82, compaction requested=true
2024-12-16T00:16:40,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for f4d6c0aff037d3eb34d03175b6108e70:

2024-12-16T00:16:40,472 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=83.1 K, sizeToCheck=16.0 K
2024-12-16T00:16:40,472 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:16:40,472 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/b7fe472d429c45b09d5e9401ec54147a because midkey is the same as first or last row
2024-12-16T00:16:40,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f4d6c0aff037d3eb34d03175b6108e70:info, priority=-2147483648, current under compaction store size is 1
2024-12-16T00:16:40,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:16:40,472 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-16T00:16:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,472 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing f4d6c0aff037d3eb34d03175b6108e70 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-12-16T00:16:40,473 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 85094 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-16T00:16:40,473 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1540): f4d6c0aff037d3eb34d03175b6108e70/info is initiating minor compaction (all files)
2024-12-16T00:16:40,473 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of f4d6c0aff037d3eb34d03175b6108e70/info in TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:40,473 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/b7fe472d429c45b09d5e9401ec54147a, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/d09019b90dd1406aa38b63962ddaa8aa, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/a19e4c58dc98405e915763a7cadcd65e] into tmpdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp, totalSize=83.1 K
2024-12-16T00:16:40,474 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting b7fe472d429c45b09d5e9401ec54147a, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1734308186271
2024-12-16T00:16:40,474 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting d09019b90dd1406aa38b63962ddaa8aa, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1734308198398
2024-12-16T00:16:40,474 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting a19e4c58dc98405e915763a7cadcd65e, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1734308198419
2024-12-16T00:16:40,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/474463326e624a91a8633a95e1dfd556 is 1080, key is row0066/info:/1734308200447/Put/seqid=0
2024-12-16T00:16:40,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741845_1021 (size=24376)
2024-12-16T00:16:40,481 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741845_1021 (size=24376)
2024-12-16T00:16:40,483 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/474463326e624a91a8633a95e1dfd556
2024-12-16T00:16:40,490 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f4d6c0aff037d3eb34d03175b6108e70#info#compaction#48 average throughput is 33.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:16:40,491 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/53429a8b29a04f8c8a38fa2bdece6bfc is 1080, key is row0001/info:/1734308186271/Put/seqid=0
2024-12-16T00:16:40,492 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f4d6c0aff037d3eb34d03175b6108e70, server=b878af803003,42285,1734308174608
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-16T00:16:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:47384 deadline: 1734308210492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=f4d6c0aff037d3eb34d03175b6108e70, server=b878af803003,42285,1734308174608
2024-12-16T00:16:40,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/474463326e624a91a8633a95e1dfd556 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/474463326e624a91a8633a95e1dfd556
2024-12-16T00:16:40,499 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/474463326e624a91a8633a95e1dfd556, entries=18, sequenceid=103, filesize=23.8 K
2024-12-16T00:16:40,500 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=11.56 KB/11836 for f4d6c0aff037d3eb34d03175b6108e70 in 28ms, sequenceid=103, compaction requested=false
2024-12-16T00:16:40,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for f4d6c0aff037d3eb34d03175b6108e70:

2024-12-16T00:16:40,500 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=106.9 K, sizeToCheck=16.0 K
2024-12-16T00:16:40,500 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:16:40,500 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/b7fe472d429c45b09d5e9401ec54147a because midkey is the same as first or last row
2024-12-16T00:16:40,504 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741846_1022 (size=75378)
2024-12-16T00:16:40,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741846_1022 (size=75378)
2024-12-16T00:16:40,510 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/53429a8b29a04f8c8a38fa2bdece6bfc as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/53429a8b29a04f8c8a38fa2bdece6bfc
2024-12-16T00:16:40,516 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in f4d6c0aff037d3eb34d03175b6108e70/info of f4d6c0aff037d3eb34d03175b6108e70 into 53429a8b29a04f8c8a38fa2bdece6bfc(size=73.6 K), total size for store is 97.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-16T00:16:40,516 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for f4d6c0aff037d3eb34d03175b6108e70:

2024-12-16T00:16:40,516 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70., storeName=f4d6c0aff037d3eb34d03175b6108e70/info, priority=13, startTime=1734308200472; duration=0sec
2024-12-16T00:16:40,516 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=97.4 K, sizeToCheck=16.0 K
2024-12-16T00:16:40,516 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-16T00:16:40,517 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:16:40,517 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:16:40,517 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f4d6c0aff037d3eb34d03175b6108e70:info
2024-12-16T00:16:40,518 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44525 {}] assignment.AssignmentManager(1346): Split request from b878af803003,42285,1734308174608, parent={ENCODED => f4d6c0aff037d3eb34d03175b6108e70, NAME => 'TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-16T00:16:40,522 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44525 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=b878af803003,42285,1734308174608
2024-12-16T00:16:40,525 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44525 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=f4d6c0aff037d3eb34d03175b6108e70, daughterA=9be904cf4ade01c9c8b0eed740afacf6, daughterB=9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:16:40,526 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=f4d6c0aff037d3eb34d03175b6108e70, daughterA=9be904cf4ade01c9c8b0eed740afacf6, daughterB=9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:16:40,526 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=f4d6c0aff037d3eb34d03175b6108e70, daughterA=9be904cf4ade01c9c8b0eed740afacf6, daughterB=9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:16:40,526 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=f4d6c0aff037d3eb34d03175b6108e70, daughterA=9be904cf4ade01c9c8b0eed740afacf6, daughterB=9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:16:40,532 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f4d6c0aff037d3eb34d03175b6108e70, UNASSIGN}]
2024-12-16T00:16:40,533 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f4d6c0aff037d3eb34d03175b6108e70, UNASSIGN
2024-12-16T00:16:40,534 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=f4d6c0aff037d3eb34d03175b6108e70, regionState=CLOSING, regionLocation=b878af803003,42285,1734308174608
2024-12-16T00:16:40,535 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-12-16T00:16:40,536 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; CloseRegionProcedure f4d6c0aff037d3eb34d03175b6108e70, server=b878af803003,42285,1734308174608}]
2024-12-16T00:16:40,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:40,692 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to b878af803003,42285,1734308174608
2024-12-16T00:16:40,693 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(124): Close f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,694 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(138): Unassign region: split region: true: evictCache: true
2024-12-16T00:16:40,694 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1681): Closing f4d6c0aff037d3eb34d03175b6108e70, disabling compactions & flushes
2024-12-16T00:16:40,694 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:40,695 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:40,695 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70. after waiting 0 ms
2024-12-16T00:16:40,695 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:40,695 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(2837): Flushing f4d6c0aff037d3eb34d03175b6108e70 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-12-16T00:16:40,700 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/7946629a755740a7be6a1ef888d60c50 is 1080, key is row0084/info:/1734308200473/Put/seqid=0
2024-12-16T00:16:40,707 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741847_1023 (size=16817)
2024-12-16T00:16:40,707 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741847_1023 (size=16817)
2024-12-16T00:16:40,708 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/7946629a755740a7be6a1ef888d60c50
2024-12-16T00:16:40,716 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/.tmp/info/7946629a755740a7be6a1ef888d60c50 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/7946629a755740a7be6a1ef888d60c50
2024-12-16T00:16:40,722 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/7946629a755740a7be6a1ef888d60c50, entries=11, sequenceid=118, filesize=16.4 K
2024-12-16T00:16:40,723 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(3040): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=0 B/0 for f4d6c0aff037d3eb34d03175b6108e70 in 27ms, sequenceid=118, compaction requested=true
2024-12-16T00:16:40,723 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/5b832ded383c4a5ca2b0e60e372ebe36, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/2be16d0b88d843e39abc7d29907bd162, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/b7fe472d429c45b09d5e9401ec54147a, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/604c62ce05794f868b3f74a47e3770cb, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/d09019b90dd1406aa38b63962ddaa8aa, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/a19e4c58dc98405e915763a7cadcd65e] to archive
2024-12-16T00:16:40,724 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-16T00:16:40,726 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/5b832ded383c4a5ca2b0e60e372ebe36 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/5b832ded383c4a5ca2b0e60e372ebe36
2024-12-16T00:16:40,727 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/2be16d0b88d843e39abc7d29907bd162 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/2be16d0b88d843e39abc7d29907bd162
2024-12-16T00:16:40,727 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/b7fe472d429c45b09d5e9401ec54147a to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/b7fe472d429c45b09d5e9401ec54147a
2024-12-16T00:16:40,727 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/d09019b90dd1406aa38b63962ddaa8aa to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/d09019b90dd1406aa38b63962ddaa8aa
2024-12-16T00:16:40,727 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/604c62ce05794f868b3f74a47e3770cb to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/604c62ce05794f868b3f74a47e3770cb
2024-12-16T00:16:40,727 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/a19e4c58dc98405e915763a7cadcd65e to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/a19e4c58dc98405e915763a7cadcd65e
2024-12-16T00:16:40,731 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=1
2024-12-16T00:16:40,732 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.
2024-12-16T00:16:40,732 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1635): Region close journal for f4d6c0aff037d3eb34d03175b6108e70:

2024-12-16T00:16:40,733 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(170): Closed f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,734 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=f4d6c0aff037d3eb34d03175b6108e70, regionState=CLOSED
2024-12-16T00:16:40,737 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13
2024-12-16T00:16:40,737 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; CloseRegionProcedure f4d6c0aff037d3eb34d03175b6108e70, server=b878af803003,42285,1734308174608 in 200 msec
2024-12-16T00:16:40,738 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12
2024-12-16T00:16:40,738 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f4d6c0aff037d3eb34d03175b6108e70, UNASSIGN in 205 msec
2024-12-16T00:16:40,757 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:40,758 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=12 splitting 3 storefiles, region=f4d6c0aff037d3eb34d03175b6108e70, threads=3
2024-12-16T00:16:40,759 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/474463326e624a91a8633a95e1dfd556 for region: f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,759 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/53429a8b29a04f8c8a38fa2bdece6bfc for region: f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,759 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/7946629a755740a7be6a1ef888d60c50 for region: f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,769 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/7946629a755740a7be6a1ef888d60c50, top=true
2024-12-16T00:16:40,770 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/474463326e624a91a8633a95e1dfd556, top=true
2024-12-16T00:16:40,779 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-474463326e624a91a8633a95e1dfd556 for child: 9b52be8ec131a4a57b605ce3472e9bf6, parent: f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,779 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-7946629a755740a7be6a1ef888d60c50 for child: 9b52be8ec131a4a57b605ce3472e9bf6, parent: f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,779 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/7946629a755740a7be6a1ef888d60c50 for region: f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,779 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/474463326e624a91a8633a95e1dfd556 for region: f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741848_1024 (size=27)
2024-12-16T00:16:40,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741848_1024 (size=27)
2024-12-16T00:16:40,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741849_1025 (size=27)
2024-12-16T00:16:40,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741849_1025 (size=27)
2024-12-16T00:16:40,791 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/53429a8b29a04f8c8a38fa2bdece6bfc for region: f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:16:40,791 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=12 split storefiles for region f4d6c0aff037d3eb34d03175b6108e70 Daughter A: [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70] storefiles, Daughter B: [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-474463326e624a91a8633a95e1dfd556, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-7946629a755740a7be6a1ef888d60c50] storefiles.
2024-12-16T00:16:40,798 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741850_1026 (size=71)
2024-12-16T00:16:40,799 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741850_1026 (size=71)
2024-12-16T00:16:40,801 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:40,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741851_1027 (size=71)
2024-12-16T00:16:40,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741851_1027 (size=71)
2024-12-16T00:16:40,812 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:40,822 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=-1
2024-12-16T00:16:40,825 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=-1
2024-12-16T00:16:40,826 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1734308200826"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1734308200826"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1734308200826"}]},"ts":"1734308200826"}
2024-12-16T00:16:40,827 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1734308200826"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734308200826"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1734308200826"}]},"ts":"1734308200826"}
2024-12-16T00:16:40,827 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1734308200826"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734308200826"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1734308200826"}]},"ts":"1734308200826"}
2024-12-16T00:16:40,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42285 {}] regionserver.HRegion(8581): Flush requested on 1588230740
2024-12-16T00:16:40,854 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-16T00:16:40,854 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=4.75 KB heapSize=8.29 KB
2024-12-16T00:16:40,858 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9be904cf4ade01c9c8b0eed740afacf6, ASSIGN}, {pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9b52be8ec131a4a57b605ce3472e9bf6, ASSIGN}]
2024-12-16T00:16:40,859 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9b52be8ec131a4a57b605ce3472e9bf6, ASSIGN
2024-12-16T00:16:40,859 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9be904cf4ade01c9c8b0eed740afacf6, ASSIGN
2024-12-16T00:16:40,860 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9b52be8ec131a4a57b605ce3472e9bf6, ASSIGN; state=SPLITTING_NEW, location=b878af803003,42285,1734308174608; forceNewPlan=false, retain=false
2024-12-16T00:16:40,860 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9be904cf4ade01c9c8b0eed740afacf6, ASSIGN; state=SPLITTING_NEW, location=b878af803003,42285,1734308174608; forceNewPlan=false, retain=false
2024-12-16T00:16:40,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/.tmp/info/9da748366aa34ddb94ccaae097de426a is 193, key is TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6./info:regioninfo/1734308200826/Put/seqid=0
2024-12-16T00:16:40,874 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741852_1028 (size=9423)
2024-12-16T00:16:40,874 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741852_1028 (size=9423)
2024-12-16T00:16:40,875 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.54 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/.tmp/info/9da748366aa34ddb94ccaae097de426a
2024-12-16T00:16:40,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/.tmp/table/012f9cd136484ea9b904cf784c73e66c is 65, key is TestLogRolling-testLogRolling/table:state/1734308176640/Put/seqid=0
2024-12-16T00:16:40,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741853_1029 (size=5412)
2024-12-16T00:16:40,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741853_1029 (size=5412)
2024-12-16T00:16:40,900 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=216 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/.tmp/table/012f9cd136484ea9b904cf784c73e66c
2024-12-16T00:16:40,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/.tmp/info/9da748366aa34ddb94ccaae097de426a as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/info/9da748366aa34ddb94ccaae097de426a
2024-12-16T00:16:40,913 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/info/9da748366aa34ddb94ccaae097de426a, entries=29, sequenceid=17, filesize=9.2 K
2024-12-16T00:16:40,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/.tmp/table/012f9cd136484ea9b904cf784c73e66c as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/table/012f9cd136484ea9b904cf784c73e66c
2024-12-16T00:16:40,919 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/table/012f9cd136484ea9b904cf784c73e66c, entries=4, sequenceid=17, filesize=5.3 K
2024-12-16T00:16:40,920 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~4.75 KB/4869, heapSize ~8.01 KB/8200, currentSize=0 B/0 for 1588230740 in 66ms, sequenceid=17, compaction requested=false
2024-12-16T00:16:40,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 1588230740:

2024-12-16T00:16:41,011 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=9be904cf4ade01c9c8b0eed740afacf6, regionState=OPENING, regionLocation=b878af803003,42285,1734308174608
2024-12-16T00:16:41,011 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=9b52be8ec131a4a57b605ce3472e9bf6, regionState=OPENING, regionLocation=b878af803003,42285,1734308174608
2024-12-16T00:16:41,012 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=15, state=RUNNABLE; OpenRegionProcedure 9be904cf4ade01c9c8b0eed740afacf6, server=b878af803003,42285,1734308174608}]
2024-12-16T00:16:41,013 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=16, state=RUNNABLE; OpenRegionProcedure 9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608}]
2024-12-16T00:16:41,164 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to b878af803003,42285,1734308174608
2024-12-16T00:16:41,168 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.
2024-12-16T00:16:41,168 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => 9be904cf4ade01c9c8b0eed740afacf6, NAME => 'TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.', STARTKEY => '', ENDKEY => 'row0062'}
2024-12-16T00:16:41,169 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 9be904cf4ade01c9c8b0eed740afacf6
2024-12-16T00:16:41,169 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:16:41,169 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for 9be904cf4ade01c9c8b0eed740afacf6
2024-12-16T00:16:41,169 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for 9be904cf4ade01c9c8b0eed740afacf6
2024-12-16T00:16:41,171 INFO  [StoreOpener-9be904cf4ade01c9c8b0eed740afacf6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9be904cf4ade01c9c8b0eed740afacf6 
2024-12-16T00:16:41,172 INFO  [StoreOpener-9be904cf4ade01c9c8b0eed740afacf6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9be904cf4ade01c9c8b0eed740afacf6 columnFamilyName info
2024-12-16T00:16:41,172 DEBUG [StoreOpener-9be904cf4ade01c9c8b0eed740afacf6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:41,184 DEBUG [StoreOpener-9be904cf4ade01c9c8b0eed740afacf6-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70->hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/53429a8b29a04f8c8a38fa2bdece6bfc-bottom
2024-12-16T00:16:41,185 INFO  [StoreOpener-9be904cf4ade01c9c8b0eed740afacf6-1 {}] regionserver.HStore(327): Store=9be904cf4ade01c9c8b0eed740afacf6/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:16:41,186 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6
2024-12-16T00:16:41,187 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6
2024-12-16T00:16:41,189 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for 9be904cf4ade01c9c8b0eed740afacf6
2024-12-16T00:16:41,189 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1102): Opened 9be904cf4ade01c9c8b0eed740afacf6; next sequenceid=122; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=695829, jitterRate=-0.11520799994468689}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:16:41,190 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for 9be904cf4ade01c9c8b0eed740afacf6:

2024-12-16T00:16:41,190 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6., pid=17, masterSystemTime=1734308201164
2024-12-16T00:16:41,190 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(403): Add compact mark for store 9be904cf4ade01c9c8b0eed740afacf6:info, priority=-2147483648, current under compaction store size is 1
2024-12-16T00:16:41,190 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:16:41,191 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-16T00:16:41,191 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.
2024-12-16T00:16:41,191 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1540): 9be904cf4ade01c9c8b0eed740afacf6/info is initiating minor compaction (all files)
2024-12-16T00:16:41,191 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9be904cf4ade01c9c8b0eed740afacf6/info in TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.
2024-12-16T00:16:41,192 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70->hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/53429a8b29a04f8c8a38fa2bdece6bfc-bottom] into tmpdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/.tmp, totalSize=73.6 K
2024-12-16T00:16:41,192 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.
2024-12-16T00:16:41,192 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.
2024-12-16T00:16:41,192 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1734308186271
2024-12-16T00:16:41,193 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=9be904cf4ade01c9c8b0eed740afacf6, regionState=OPEN, openSeqNum=122, regionLocation=b878af803003,42285,1734308174608
2024-12-16T00:16:41,193 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:16:41,193 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7285): Opening region: {ENCODED => 9b52be8ec131a4a57b605ce3472e9bf6, NAME => 'TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.', STARTKEY => 'row0062', ENDKEY => ''}
2024-12-16T00:16:41,193 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:16:41,193 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:16:41,193 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7327): checking encryption for 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:16:41,193 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7330): checking classloading for 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:16:41,195 INFO  [StoreOpener-9b52be8ec131a4a57b605ce3472e9bf6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9b52be8ec131a4a57b605ce3472e9bf6 
2024-12-16T00:16:41,197 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=15
2024-12-16T00:16:41,197 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=15, state=SUCCESS; OpenRegionProcedure 9be904cf4ade01c9c8b0eed740afacf6, server=b878af803003,42285,1734308174608 in 183 msec
2024-12-16T00:16:41,199 INFO  [StoreOpener-9b52be8ec131a4a57b605ce3472e9bf6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b52be8ec131a4a57b605ce3472e9bf6 columnFamilyName info
2024-12-16T00:16:41,199 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9be904cf4ade01c9c8b0eed740afacf6, ASSIGN in 339 msec
2024-12-16T00:16:41,199 DEBUG [StoreOpener-9b52be8ec131a4a57b605ce3472e9bf6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:16:41,213 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9be904cf4ade01c9c8b0eed740afacf6#info#compaction#52 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:16:41,214 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/.tmp/info/1ed2f8c4b4dd40418312a954479e1534 is 1080, key is row0001/info:/1734308186271/Put/seqid=0
2024-12-16T00:16:41,217 DEBUG [StoreOpener-9b52be8ec131a4a57b605ce3472e9bf6-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70->hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/53429a8b29a04f8c8a38fa2bdece6bfc-top
2024-12-16T00:16:41,221 DEBUG [StoreOpener-9b52be8ec131a4a57b605ce3472e9bf6-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-474463326e624a91a8633a95e1dfd556
2024-12-16T00:16:41,226 DEBUG [StoreOpener-9b52be8ec131a4a57b605ce3472e9bf6-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-7946629a755740a7be6a1ef888d60c50
2024-12-16T00:16:41,226 INFO  [StoreOpener-9b52be8ec131a4a57b605ce3472e9bf6-1 {}] regionserver.HStore(327): Store=9b52be8ec131a4a57b605ce3472e9bf6/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:16:41,227 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:16:41,228 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:16:41,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741854_1030 (size=70862)
2024-12-16T00:16:41,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741854_1030 (size=70862)
2024-12-16T00:16:41,231 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1085): writing seq id for 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:16:41,232 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1102): Opened 9b52be8ec131a4a57b605ce3472e9bf6; next sequenceid=122; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=803816, jitterRate=0.022105351090431213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:16:41,232 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1001): Region open journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:16:41,233 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6., pid=18, masterSystemTime=1734308201164
2024-12-16T00:16:41,233 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(403): Add compact mark for store 9b52be8ec131a4a57b605ce3472e9bf6:info, priority=-2147483648, current under compaction store size is 2
2024-12-16T00:16:41,233 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0
2024-12-16T00:16:41,233 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-16T00:16:41,234 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:16:41,235 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HStore(1540): 9b52be8ec131a4a57b605ce3472e9bf6/info is initiating minor compaction (all files)
2024-12-16T00:16:41,235 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b52be8ec131a4a57b605ce3472e9bf6/info in TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:16:41,235 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70->hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/53429a8b29a04f8c8a38fa2bdece6bfc-top, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-474463326e624a91a8633a95e1dfd556, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-7946629a755740a7be6a1ef888d60c50] into tmpdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp, totalSize=113.8 K
2024-12-16T00:16:41,235 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.Compactor(224): Compacting 53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1734308186271
2024-12-16T00:16:41,236 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.Compactor(224): Compacting TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-474463326e624a91a8633a95e1dfd556, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1734308200447
2024-12-16T00:16:41,236 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.Compactor(224): Compacting TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-7946629a755740a7be6a1ef888d60c50, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1734308200473
2024-12-16T00:16:41,238 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/.tmp/info/1ed2f8c4b4dd40418312a954479e1534 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/info/1ed2f8c4b4dd40418312a954479e1534
2024-12-16T00:16:41,245 DEBUG [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:16:41,245 INFO  [RS_OPEN_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:16:41,247 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=9b52be8ec131a4a57b605ce3472e9bf6, regionState=OPEN, openSeqNum=122, regionLocation=b878af803003,42285,1734308174608
2024-12-16T00:16:41,249 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 1 (all) file(s) in 9be904cf4ade01c9c8b0eed740afacf6/info of 9be904cf4ade01c9c8b0eed740afacf6 into 1ed2f8c4b4dd40418312a954479e1534(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-16T00:16:41,250 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9be904cf4ade01c9c8b0eed740afacf6:

2024-12-16T00:16:41,250 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6., storeName=9be904cf4ade01c9c8b0eed740afacf6/info, priority=15, startTime=1734308201190; duration=0sec
2024-12-16T00:16:41,250 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:16:41,250 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9be904cf4ade01c9c8b0eed740afacf6:info
2024-12-16T00:16:41,251 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=16
2024-12-16T00:16:41,251 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=16, state=SUCCESS; OpenRegionProcedure 9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608 in 235 msec
2024-12-16T00:16:41,253 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=12
2024-12-16T00:16:41,253 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9b52be8ec131a4a57b605ce3472e9bf6, ASSIGN in 393 msec
2024-12-16T00:16:41,254 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=f4d6c0aff037d3eb34d03175b6108e70, daughterA=9be904cf4ade01c9c8b0eed740afacf6, daughterB=9b52be8ec131a4a57b605ce3472e9bf6 in 731 msec
2024-12-16T00:16:41,262 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b52be8ec131a4a57b605ce3472e9bf6#info#compaction#53 average throughput is 33.86 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:16:41,263 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/6d1243e026e94b1384cb46fafaf623fa is 1080, key is row0062/info:/1734308198440/Put/seqid=0
2024-12-16T00:16:41,269 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741855_1031 (size=40830)
2024-12-16T00:16:41,269 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741855_1031 (size=40830)
2024-12-16T00:16:41,283 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/6d1243e026e94b1384cb46fafaf623fa as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/6d1243e026e94b1384cb46fafaf623fa
2024-12-16T00:16:41,290 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b52be8ec131a4a57b605ce3472e9bf6/info of 9b52be8ec131a4a57b605ce3472e9bf6 into 6d1243e026e94b1384cb46fafaf623fa(size=39.9 K), total size for store is 39.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-16T00:16:41,290 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:16:41,290 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6., storeName=9b52be8ec131a4a57b605ce3472e9bf6/info, priority=13, startTime=1734308201233; duration=0sec
2024-12-16T00:16:41,290 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:16:41,291 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b52be8ec131a4a57b605ce3472e9bf6:info
2024-12-16T00:16:41,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:42,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:43,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:44,444 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-16T00:16:44,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:45,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:45,733 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,734 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,734 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,734 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,753 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,753 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,756 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,756 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,756 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:45,758 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,264 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-16T00:16:46,265 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,266 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,267 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,290 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,290 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,291 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,291 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,291 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,291 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,294 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,294 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,295 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,297 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-16T00:16:46,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:47,571 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:48,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:48,956 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:16:48,960 INFO  [RS-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46728, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:16:49,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:50,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:47384 deadline: 1734308220601, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1734308176252.f4d6c0aff037d3eb34d03175b6108e70. is not online on b878af803003,42285,1734308174608
2024-12-16T00:16:51,576 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:52,577 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:53,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:54,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:55,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:56,422 INFO  [master/b878af803003:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-16T00:16:56,423 INFO  [master/b878af803003:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-16T00:16:56,581 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:57,582 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:58,583 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:16:59,583 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:00,500 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 14835
2024-12-16T00:17:00,584 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:01,585 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:02,586 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:03,586 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:04,588 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:05,589 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:06,590 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:07,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:08,592 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:09,592 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:10,593 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:11,594 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:12,594 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:17:12,741 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-16T00:17:12,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/99185f634a654216b50d7ee8ac3febdd is 1080, key is row0095/info:/1734308230732/Put/seqid=0
2024-12-16T00:17:12,753 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741856_1032 (size=12513)
2024-12-16T00:17:12,753 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741856_1032 (size=12513)
2024-12-16T00:17:12,754 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/99185f634a654216b50d7ee8ac3febdd
2024-12-16T00:17:12,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/99185f634a654216b50d7ee8ac3febdd as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/99185f634a654216b50d7ee8ac3febdd
2024-12-16T00:17:12,765 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/99185f634a654216b50d7ee8ac3febdd, entries=7, sequenceid=132, filesize=12.2 K
2024-12-16T00:17:12,766 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for 9b52be8ec131a4a57b605ce3472e9bf6 in 25ms, sequenceid=132, compaction requested=false
2024-12-16T00:17:12,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:17:12,767 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=19.96 KB heapSize=21.63 KB
2024-12-16T00:17:12,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/19e01ea6eadb49d1a33f5280b309efaf is 1080, key is row0102/info:/1734308232742/Put/seqid=0
2024-12-16T00:17:12,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741857_1033 (size=25472)
2024-12-16T00:17:12,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741857_1033 (size=25472)
2024-12-16T00:17:12,781 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/19e01ea6eadb49d1a33f5280b309efaf
2024-12-16T00:17:12,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/19e01ea6eadb49d1a33f5280b309efaf as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/19e01ea6eadb49d1a33f5280b309efaf
2024-12-16T00:17:12,795 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/19e01ea6eadb49d1a33f5280b309efaf, entries=19, sequenceid=154, filesize=24.9 K
2024-12-16T00:17:12,797 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~19.96 KB/20444, heapSize ~21.61 KB/22128, currentSize=8.41 KB/8608 for 9b52be8ec131a4a57b605ce3472e9bf6 in 29ms, sequenceid=154, compaction requested=true
2024-12-16T00:17:12,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:12,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b52be8ec131a4a57b605ce3472e9bf6:info, priority=-2147483648, current under compaction store size is 1
2024-12-16T00:17:12,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:17:12,797 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-16T00:17:12,798 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 78815 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-16T00:17:12,798 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1540): 9b52be8ec131a4a57b605ce3472e9bf6/info is initiating minor compaction (all files)
2024-12-16T00:17:12,798 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b52be8ec131a4a57b605ce3472e9bf6/info in TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:17:12,798 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/6d1243e026e94b1384cb46fafaf623fa, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/99185f634a654216b50d7ee8ac3febdd, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/19e01ea6eadb49d1a33f5280b309efaf] into tmpdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp, totalSize=77.0 K
2024-12-16T00:17:12,799 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6d1243e026e94b1384cb46fafaf623fa, keycount=33, bloomtype=ROW, size=39.9 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1734308198440
2024-12-16T00:17:12,800 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 99185f634a654216b50d7ee8ac3febdd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1734308230732
2024-12-16T00:17:12,800 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 19e01ea6eadb49d1a33f5280b309efaf, keycount=19, bloomtype=ROW, size=24.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1734308232742
2024-12-16T00:17:12,814 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b52be8ec131a4a57b605ce3472e9bf6#info#compaction#56 average throughput is 30.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:17:12,815 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/8a78f64ecffb4b5e84e2056bbf55de68 is 1080, key is row0062/info:/1734308198440/Put/seqid=0
2024-12-16T00:17:12,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741858_1034 (size=69025)
2024-12-16T00:17:12,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741858_1034 (size=69025)
2024-12-16T00:17:12,824 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/8a78f64ecffb4b5e84e2056bbf55de68 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8a78f64ecffb4b5e84e2056bbf55de68
2024-12-16T00:17:12,830 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b52be8ec131a4a57b605ce3472e9bf6/info of 9b52be8ec131a4a57b605ce3472e9bf6 into 8a78f64ecffb4b5e84e2056bbf55de68(size=67.4 K), total size for store is 67.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-16T00:17:12,830 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:12,830 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6., storeName=9b52be8ec131a4a57b605ce3472e9bf6/info, priority=13, startTime=1734308232797; duration=0sec
2024-12-16T00:17:12,831 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:17:12,831 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b52be8ec131a4a57b605ce3472e9bf6:info
2024-12-16T00:17:13,595 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:14,444 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-16T00:17:14,596 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:17:14,780 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-16T00:17:14,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/622e457aae374ccaabd2abb3cd668b6d is 1080, key is row0121/info:/1734308232768/Put/seqid=0
2024-12-16T00:17:14,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741859_1035 (size=14672)
2024-12-16T00:17:14,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741859_1035 (size=14672)
2024-12-16T00:17:14,797 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/622e457aae374ccaabd2abb3cd668b6d
2024-12-16T00:17:14,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/622e457aae374ccaabd2abb3cd668b6d as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/622e457aae374ccaabd2abb3cd668b6d
2024-12-16T00:17:14,807 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-16T00:17:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:47384 deadline: 1734308244807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608
2024-12-16T00:17:14,811 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/622e457aae374ccaabd2abb3cd668b6d, entries=9, sequenceid=167, filesize=14.3 K
2024-12-16T00:17:14,812 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=21.02 KB/21520 for 9b52be8ec131a4a57b605ce3472e9bf6 in 32ms, sequenceid=167, compaction requested=false
2024-12-16T00:17:14,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:15,596 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:15,598 DEBUG [master/b878af803003:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 5982437be6e272746f8b60f107f87ef4 changed from -1.0 to 0.0, refreshing cache
2024-12-16T00:17:16,597 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:17,598 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:18,599 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:19,600 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:20,601 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:21,601 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:22,602 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:23,602 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:24,603 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:17:24,859 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=22.07 KB heapSize=23.88 KB
2024-12-16T00:17:24,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/fd0f6fc0e5df4b2fb530275ea0d8996f is 1080, key is row0130/info:/1734308234781/Put/seqid=0
2024-12-16T00:17:24,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741860_1036 (size=27628)
2024-12-16T00:17:24,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741860_1036 (size=27628)
2024-12-16T00:17:24,873 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/fd0f6fc0e5df4b2fb530275ea0d8996f
2024-12-16T00:17:24,875 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-16T00:17:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:47384 deadline: 1734308254875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608
2024-12-16T00:17:24,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/fd0f6fc0e5df4b2fb530275ea0d8996f as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/fd0f6fc0e5df4b2fb530275ea0d8996f
2024-12-16T00:17:24,885 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/fd0f6fc0e5df4b2fb530275ea0d8996f, entries=21, sequenceid=191, filesize=27.0 K
2024-12-16T00:17:24,886 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~22.07 KB/22596, heapSize ~23.86 KB/24432, currentSize=8.41 KB/8608 for 9b52be8ec131a4a57b605ce3472e9bf6 in 28ms, sequenceid=191, compaction requested=true
2024-12-16T00:17:24,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:24,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b52be8ec131a4a57b605ce3472e9bf6:info, priority=-2147483648, current under compaction store size is 1
2024-12-16T00:17:24,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:17:24,886 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-16T00:17:24,887 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 111325 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-16T00:17:24,887 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HStore(1540): 9b52be8ec131a4a57b605ce3472e9bf6/info is initiating minor compaction (all files)
2024-12-16T00:17:24,887 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b52be8ec131a4a57b605ce3472e9bf6/info in TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:17:24,887 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8a78f64ecffb4b5e84e2056bbf55de68, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/622e457aae374ccaabd2abb3cd668b6d, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/fd0f6fc0e5df4b2fb530275ea0d8996f] into tmpdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp, totalSize=108.7 K
2024-12-16T00:17:24,888 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.Compactor(224): Compacting 8a78f64ecffb4b5e84e2056bbf55de68, keycount=59, bloomtype=ROW, size=67.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1734308198440
2024-12-16T00:17:24,888 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.Compactor(224): Compacting 622e457aae374ccaabd2abb3cd668b6d, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1734308232768
2024-12-16T00:17:24,888 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.Compactor(224): Compacting fd0f6fc0e5df4b2fb530275ea0d8996f, keycount=21, bloomtype=ROW, size=27.0 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1734308234781
2024-12-16T00:17:24,900 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b52be8ec131a4a57b605ce3472e9bf6#info#compaction#59 average throughput is 30.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:17:24,901 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/9942f1db6b544068a26c8dc13646d9f0 is 1080, key is row0062/info:/1734308198440/Put/seqid=0
2024-12-16T00:17:24,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741861_1037 (size=101544)
2024-12-16T00:17:24,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741861_1037 (size=101544)
2024-12-16T00:17:24,911 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/9942f1db6b544068a26c8dc13646d9f0 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/9942f1db6b544068a26c8dc13646d9f0
2024-12-16T00:17:24,916 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b52be8ec131a4a57b605ce3472e9bf6/info of 9b52be8ec131a4a57b605ce3472e9bf6 into 9942f1db6b544068a26c8dc13646d9f0(size=99.2 K), total size for store is 99.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-16T00:17:24,916 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:24,916 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6., storeName=9b52be8ec131a4a57b605ce3472e9bf6/info, priority=13, startTime=1734308244886; duration=0sec
2024-12-16T00:17:24,916 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:17:24,916 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b52be8ec131a4a57b605ce3472e9bf6:info
2024-12-16T00:17:25,604 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:25,604 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=3 on file=hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta after 196221ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor199.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-16T00:17:26,169 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 9be904cf4ade01c9c8b0eed740afacf6, had cached 0 bytes from a total of 70862
2024-12-16T00:17:26,193 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 9b52be8ec131a4a57b605ce3472e9bf6, had cached 0 bytes from a total of 101544
2024-12-16T00:17:26,605 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:27,606 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:28,607 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:29,608 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:30,609 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:31,609 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:32,610 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:33,612 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:34,613 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:17:34,895 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-16T00:17:34,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/e463f8502b2743cda9f0468bb54b5b23 is 1080, key is row0151/info:/1734308244860/Put/seqid=0
2024-12-16T00:17:34,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741862_1038 (size=14672)
2024-12-16T00:17:34,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741862_1038 (size=14672)
2024-12-16T00:17:34,904 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/e463f8502b2743cda9f0468bb54b5b23
2024-12-16T00:17:34,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/e463f8502b2743cda9f0468bb54b5b23 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/e463f8502b2743cda9f0468bb54b5b23
2024-12-16T00:17:34,914 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/e463f8502b2743cda9f0468bb54b5b23, entries=9, sequenceid=204, filesize=14.3 K
2024-12-16T00:17:34,915 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=1.05 KB/1076 for 9b52be8ec131a4a57b605ce3472e9bf6 in 20ms, sequenceid=204, compaction requested=false
2024-12-16T00:17:34,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:35,613 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:36,614 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:17:36,905 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-16T00:17:36,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/8836841ff1424523b03b5a81f650f754 is 1080, key is row0160/info:/1734308254896/Put/seqid=0
2024-12-16T00:17:36,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741863_1039 (size=12516)
2024-12-16T00:17:36,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741863_1039 (size=12516)
2024-12-16T00:17:36,915 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/8836841ff1424523b03b5a81f650f754
2024-12-16T00:17:36,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/8836841ff1424523b03b5a81f650f754 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8836841ff1424523b03b5a81f650f754
2024-12-16T00:17:36,938 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8836841ff1424523b03b5a81f650f754, entries=7, sequenceid=214, filesize=12.2 K
2024-12-16T00:17:36,938 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-16T00:17:36,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:47384 deadline: 1734308266938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608
2024-12-16T00:17:36,939 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 9b52be8ec131a4a57b605ce3472e9bf6 in 35ms, sequenceid=214, compaction requested=true
2024-12-16T00:17:36,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:36,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b52be8ec131a4a57b605ce3472e9bf6:info, priority=-2147483648, current under compaction store size is 1
2024-12-16T00:17:36,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:17:36,939 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-16T00:17:36,940 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 128732 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-16T00:17:36,940 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1540): 9b52be8ec131a4a57b605ce3472e9bf6/info is initiating minor compaction (all files)
2024-12-16T00:17:36,940 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b52be8ec131a4a57b605ce3472e9bf6/info in TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:17:36,941 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/9942f1db6b544068a26c8dc13646d9f0, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/e463f8502b2743cda9f0468bb54b5b23, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8836841ff1424523b03b5a81f650f754] into tmpdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp, totalSize=125.7 K
2024-12-16T00:17:36,941 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9942f1db6b544068a26c8dc13646d9f0, keycount=89, bloomtype=ROW, size=99.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1734308198440
2024-12-16T00:17:36,941 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting e463f8502b2743cda9f0468bb54b5b23, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1734308244860
2024-12-16T00:17:36,942 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8836841ff1424523b03b5a81f650f754, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1734308254896
2024-12-16T00:17:36,955 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b52be8ec131a4a57b605ce3472e9bf6#info#compaction#62 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:17:36,955 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/2dfa4452a492435fb11110ad43394416 is 1080, key is row0062/info:/1734308198440/Put/seqid=0
2024-12-16T00:17:36,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741864_1040 (size=118898)
2024-12-16T00:17:36,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741864_1040 (size=118898)
2024-12-16T00:17:36,964 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/2dfa4452a492435fb11110ad43394416 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/2dfa4452a492435fb11110ad43394416
2024-12-16T00:17:36,969 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b52be8ec131a4a57b605ce3472e9bf6/info of 9b52be8ec131a4a57b605ce3472e9bf6 into 2dfa4452a492435fb11110ad43394416(size=116.1 K), total size for store is 116.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-16T00:17:36,969 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:36,969 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6., storeName=9b52be8ec131a4a57b605ce3472e9bf6/info, priority=13, startTime=1734308256939; duration=0sec
2024-12-16T00:17:36,970 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:17:36,970 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b52be8ec131a4a57b605ce3472e9bf6:info
2024-12-16T00:17:37,615 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:38,616 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:39,616 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:40,617 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:41,617 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:42,618 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:43,619 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:44,445 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-16T00:17:44,619 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:45,500 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 14835
2024-12-16T00:17:45,620 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:46,621 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:17:46,976 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-16T00:17:46,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/f033133eb95346ed99833dca626a1a97 is 1080, key is row0167/info:/1734308256905/Put/seqid=0
2024-12-16T00:17:46,988 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741865_1041 (size=29784)
2024-12-16T00:17:46,988 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741865_1041 (size=29784)
2024-12-16T00:17:46,989 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/f033133eb95346ed99833dca626a1a97
2024-12-16T00:17:46,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/f033133eb95346ed99833dca626a1a97 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/f033133eb95346ed99833dca626a1a97
2024-12-16T00:17:46,998 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/f033133eb95346ed99833dca626a1a97, entries=23, sequenceid=241, filesize=29.1 K
2024-12-16T00:17:46,999 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=3.15 KB/3228 for 9b52be8ec131a4a57b605ce3472e9bf6 in 23ms, sequenceid=241, compaction requested=false
2024-12-16T00:17:46,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:47,622 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:48,623 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:17:48,990 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-16T00:17:48,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/9e86af3537ae4f448298b4efc75ff073 is 1080, key is row0190/info:/1734308266977/Put/seqid=0
2024-12-16T00:17:49,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741866_1042 (size=12516)
2024-12-16T00:17:49,004 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741866_1042 (size=12516)
2024-12-16T00:17:49,004 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/9e86af3537ae4f448298b4efc75ff073
2024-12-16T00:17:49,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/9e86af3537ae4f448298b4efc75ff073 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/9e86af3537ae4f448298b4efc75ff073
2024-12-16T00:17:49,015 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/9e86af3537ae4f448298b4efc75ff073, entries=7, sequenceid=251, filesize=12.2 K
2024-12-16T00:17:49,016 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=19.96 KB/20444 for 9b52be8ec131a4a57b605ce3472e9bf6 in 26ms, sequenceid=251, compaction requested=true
2024-12-16T00:17:49,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:49,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b52be8ec131a4a57b605ce3472e9bf6:info, priority=-2147483648, current under compaction store size is 1
2024-12-16T00:17:49,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:17:49,016 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-16T00:17:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:17:49,017 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=21.02 KB heapSize=22.75 KB
2024-12-16T00:17:49,017 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 161198 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-16T00:17:49,017 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1540): 9b52be8ec131a4a57b605ce3472e9bf6/info is initiating minor compaction (all files)
2024-12-16T00:17:49,017 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b52be8ec131a4a57b605ce3472e9bf6/info in TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:17:49,017 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/2dfa4452a492435fb11110ad43394416, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/f033133eb95346ed99833dca626a1a97, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/9e86af3537ae4f448298b4efc75ff073] into tmpdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp, totalSize=157.4 K
2024-12-16T00:17:49,018 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2dfa4452a492435fb11110ad43394416, keycount=105, bloomtype=ROW, size=116.1 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1734308198440
2024-12-16T00:17:49,018 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting f033133eb95346ed99833dca626a1a97, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1734308256905
2024-12-16T00:17:49,018 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9e86af3537ae4f448298b4efc75ff073, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1734308266977
2024-12-16T00:17:49,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/13d5c7632e914d08a79f2a57e73cc198 is 1080, key is row0197/info:/1734308268991/Put/seqid=0
2024-12-16T00:17:49,024 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741867_1043 (size=26567)
2024-12-16T00:17:49,024 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741867_1043 (size=26567)
2024-12-16T00:17:49,025 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.02 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/13d5c7632e914d08a79f2a57e73cc198
2024-12-16T00:17:49,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/13d5c7632e914d08a79f2a57e73cc198 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/13d5c7632e914d08a79f2a57e73cc198
2024-12-16T00:17:49,031 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b52be8ec131a4a57b605ce3472e9bf6#info#compaction#66 average throughput is 46.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:17:49,032 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/f742e113e90047c5813f364ef37b56db is 1080, key is row0062/info:/1734308198440/Put/seqid=0
2024-12-16T00:17:49,034 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/13d5c7632e914d08a79f2a57e73cc198, entries=20, sequenceid=274, filesize=25.9 K
2024-12-16T00:17:49,035 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~21.02 KB/21520, heapSize ~22.73 KB/23280, currentSize=8.41 KB/8608 for 9b52be8ec131a4a57b605ce3472e9bf6 in 18ms, sequenceid=274, compaction requested=false
2024-12-16T00:17:49,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:49,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741868_1044 (size=151545)
2024-12-16T00:17:49,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741868_1044 (size=151545)
2024-12-16T00:17:49,042 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/f742e113e90047c5813f364ef37b56db as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/f742e113e90047c5813f364ef37b56db
2024-12-16T00:17:49,047 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b52be8ec131a4a57b605ce3472e9bf6/info of 9b52be8ec131a4a57b605ce3472e9bf6 into f742e113e90047c5813f364ef37b56db(size=148.0 K), total size for store is 173.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-16T00:17:49,047 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:49,047 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6., storeName=9b52be8ec131a4a57b605ce3472e9bf6/info, priority=13, startTime=1734308269016; duration=0sec
2024-12-16T00:17:49,047 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:17:49,047 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b52be8ec131a4a57b605ce3472e9bf6:info
2024-12-16T00:17:49,625 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:50,626 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:17:51,032 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-16T00:17:51,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/317a9416f5cb4ec998d2ff95904a0c71 is 1080, key is row0217/info:/1734308269017/Put/seqid=0
2024-12-16T00:17:51,058 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-16T00:17:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:47384 deadline: 1734308281057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608
2024-12-16T00:17:51,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741869_1045 (size=14681)
2024-12-16T00:17:51,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741869_1045 (size=14681)
2024-12-16T00:17:51,060 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/317a9416f5cb4ec998d2ff95904a0c71
2024-12-16T00:17:51,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/317a9416f5cb4ec998d2ff95904a0c71 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/317a9416f5cb4ec998d2ff95904a0c71
2024-12-16T00:17:51,076 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/317a9416f5cb4ec998d2ff95904a0c71, entries=9, sequenceid=287, filesize=14.3 K
2024-12-16T00:17:51,077 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=21.02 KB/21520 for 9b52be8ec131a4a57b605ce3472e9bf6 in 45ms, sequenceid=287, compaction requested=true
2024-12-16T00:17:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b52be8ec131a4a57b605ce3472e9bf6:info, priority=-2147483648, current under compaction store size is 1
2024-12-16T00:17:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:17:51,077 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-16T00:17:51,078 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 192793 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-16T00:17:51,078 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1540): 9b52be8ec131a4a57b605ce3472e9bf6/info is initiating minor compaction (all files)
2024-12-16T00:17:51,078 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b52be8ec131a4a57b605ce3472e9bf6/info in TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:17:51,078 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/f742e113e90047c5813f364ef37b56db, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/13d5c7632e914d08a79f2a57e73cc198, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/317a9416f5cb4ec998d2ff95904a0c71] into tmpdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp, totalSize=188.3 K
2024-12-16T00:17:51,078 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting f742e113e90047c5813f364ef37b56db, keycount=135, bloomtype=ROW, size=148.0 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1734308198440
2024-12-16T00:17:51,079 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 13d5c7632e914d08a79f2a57e73cc198, keycount=20, bloomtype=ROW, size=25.9 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1734308268991
2024-12-16T00:17:51,079 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] compactions.Compactor(224): Compacting 317a9416f5cb4ec998d2ff95904a0c71, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1734308269017
2024-12-16T00:17:51,089 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b52be8ec131a4a57b605ce3472e9bf6#info#compaction#68 average throughput is 84.14 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:17:51,090 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/bda99fef680942e49fbc945a07bea41d is 1080, key is row0062/info:/1734308198440/Put/seqid=0
2024-12-16T00:17:51,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741870_1046 (size=182943)
2024-12-16T00:17:51,094 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741870_1046 (size=182943)
2024-12-16T00:17:51,099 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/bda99fef680942e49fbc945a07bea41d as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/bda99fef680942e49fbc945a07bea41d
2024-12-16T00:17:51,104 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b52be8ec131a4a57b605ce3472e9bf6/info of 9b52be8ec131a4a57b605ce3472e9bf6 into bda99fef680942e49fbc945a07bea41d(size=178.7 K), total size for store is 178.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-16T00:17:51,105 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:17:51,105 INFO  [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6., storeName=9b52be8ec131a4a57b605ce3472e9bf6/info, priority=13, startTime=1734308271077; duration=0sec
2024-12-16T00:17:51,105 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:17:51,105 DEBUG [RS:0;b878af803003:42285-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b52be8ec131a4a57b605ce3472e9bf6:info
2024-12-16T00:17:51,627 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:52,627 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:53,629 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:54,630 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:55,630 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:56,631 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:57,632 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:58,632 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:17:59,633 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:00,633 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:18:01,134 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=22.07 KB heapSize=23.88 KB
2024-12-16T00:18:01,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/2b714407015f4df1a4473ecb1eb65550 is 1080, key is row0226/info:/1734308271032/Put/seqid=0
2024-12-16T00:18:01,145 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-16T00:18:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:47384 deadline: 1734308291145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9b52be8ec131a4a57b605ce3472e9bf6, server=b878af803003,42285,1734308174608
2024-12-16T00:18:01,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741871_1047 (size=27649)
2024-12-16T00:18:01,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741871_1047 (size=27649)
2024-12-16T00:18:01,148 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.07 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/2b714407015f4df1a4473ecb1eb65550
2024-12-16T00:18:01,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/2b714407015f4df1a4473ecb1eb65550 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/2b714407015f4df1a4473ecb1eb65550
2024-12-16T00:18:01,160 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/2b714407015f4df1a4473ecb1eb65550, entries=21, sequenceid=312, filesize=27.0 K
2024-12-16T00:18:01,161 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~22.07 KB/22596, heapSize ~23.86 KB/24432, currentSize=8.41 KB/8608 for 9b52be8ec131a4a57b605ce3472e9bf6 in 27ms, sequenceid=312, compaction requested=false
2024-12-16T00:18:01,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:18:01,634 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:02,635 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:03,635 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:04,636 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:05,637 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:06,637 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:07,638 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:08,638 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:09,639 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:10,639 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:11,170 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 9be904cf4ade01c9c8b0eed740afacf6, had cached 0 bytes from a total of 70862
2024-12-16T00:18:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42285 {}] regionserver.HRegion(8581): Flush requested on 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:18:11,176 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-16T00:18:11,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/8de63d09f2b64a6f8af797067efc9fa6 is 1080, key is row0247/info:/1734308281134/Put/seqid=0
2024-12-16T00:18:11,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741872_1048 (size=14681)
2024-12-16T00:18:11,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741872_1048 (size=14681)
2024-12-16T00:18:11,192 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/8de63d09f2b64a6f8af797067efc9fa6
2024-12-16T00:18:11,194 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 9b52be8ec131a4a57b605ce3472e9bf6, had cached 0 bytes from a total of 210592
2024-12-16T00:18:11,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/8de63d09f2b64a6f8af797067efc9fa6 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8de63d09f2b64a6f8af797067efc9fa6
2024-12-16T00:18:11,201 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8de63d09f2b64a6f8af797067efc9fa6, entries=9, sequenceid=324, filesize=14.3 K
2024-12-16T00:18:11,202 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=1.05 KB/1076 for 9b52be8ec131a4a57b605ce3472e9bf6 in 26ms, sequenceid=324, compaction requested=true
2024-12-16T00:18:11,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:18:11,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b52be8ec131a4a57b605ce3472e9bf6:info, priority=-2147483648, current under compaction store size is 1
2024-12-16T00:18:11,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:18:11,202 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-16T00:18:11,203 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 225273 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-16T00:18:11,203 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HStore(1540): 9b52be8ec131a4a57b605ce3472e9bf6/info is initiating minor compaction (all files)
2024-12-16T00:18:11,203 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b52be8ec131a4a57b605ce3472e9bf6/info in TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:18:11,204 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/bda99fef680942e49fbc945a07bea41d, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/2b714407015f4df1a4473ecb1eb65550, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8de63d09f2b64a6f8af797067efc9fa6] into tmpdir=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp, totalSize=220.0 K
2024-12-16T00:18:11,204 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.Compactor(224): Compacting bda99fef680942e49fbc945a07bea41d, keycount=164, bloomtype=ROW, size=178.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1734308198440
2024-12-16T00:18:11,204 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.Compactor(224): Compacting 2b714407015f4df1a4473ecb1eb65550, keycount=21, bloomtype=ROW, size=27.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1734308271032
2024-12-16T00:18:11,205 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] compactions.Compactor(224): Compacting 8de63d09f2b64a6f8af797067efc9fa6, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1734308281134
2024-12-16T00:18:11,217 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b52be8ec131a4a57b605ce3472e9bf6#info#compaction#71 average throughput is 66.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-16T00:18:11,217 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/c6afe4db68d647df98b814ec3bda51b6 is 1080, key is row0062/info:/1734308198440/Put/seqid=0
2024-12-16T00:18:11,220 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741873_1049 (size=215492)
2024-12-16T00:18:11,220 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741873_1049 (size=215492)
2024-12-16T00:18:11,225 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/c6afe4db68d647df98b814ec3bda51b6 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/c6afe4db68d647df98b814ec3bda51b6
2024-12-16T00:18:11,230 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b52be8ec131a4a57b605ce3472e9bf6/info of 9b52be8ec131a4a57b605ce3472e9bf6 into c6afe4db68d647df98b814ec3bda51b6(size=210.4 K), total size for store is 210.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-16T00:18:11,230 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:18:11,230 INFO  [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6., storeName=9b52be8ec131a4a57b605ce3472e9bf6/info, priority=13, startTime=1734308291202; duration=0sec
2024-12-16T00:18:11,230 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-16T00:18:11,230 DEBUG [RS:0;b878af803003:42285-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b52be8ec131a4a57b605ce3472e9bf6:info
2024-12-16T00:18:11,640 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:12,641 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:13,178 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-12-16T00:18:13,179 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C42285%2C1734308174608.1734308293178
2024-12-16T00:18:13,189 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608/b878af803003%2C42285%2C1734308174608.1734308175117 with entries=311, filesize=307.59 KB; new WAL /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608/b878af803003%2C42285%2C1734308174608.1734308293178
2024-12-16T00:18:13,190 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43035:43035),(127.0.0.1/127.0.0.1:37683:37683)]
2024-12-16T00:18:13,190 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608/b878af803003%2C42285%2C1734308174608.1734308175117 is not closed yet, will try archiving it next time
2024-12-16T00:18:13,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741833_1009 (size=314980)
2024-12-16T00:18:13,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741833_1009 (size=314980)
2024-12-16T00:18:13,193 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.21 KB heapSize=4.13 KB
2024-12-16T00:18:13,197 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/.tmp/info/45f5713b11164e9eaf1d95abc0924fe3 is 193, key is TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6./info:regioninfo/1734308201246/Put/seqid=0
2024-12-16T00:18:13,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741875_1051 (size=7803)
2024-12-16T00:18:13,215 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741875_1051 (size=7803)
2024-12-16T00:18:13,215 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.21 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/.tmp/info/45f5713b11164e9eaf1d95abc0924fe3
2024-12-16T00:18:13,225 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/.tmp/info/45f5713b11164e9eaf1d95abc0924fe3 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/info/45f5713b11164e9eaf1d95abc0924fe3
2024-12-16T00:18:13,232 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/info/45f5713b11164e9eaf1d95abc0924fe3, entries=16, sequenceid=24, filesize=7.6 K
2024-12-16T00:18:13,233 INFO  [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~2.21 KB/2260, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 39ms, sequenceid=24, compaction requested=false
2024-12-16T00:18:13,233 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740:

2024-12-16T00:18:13,233 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 5982437be6e272746f8b60f107f87ef4 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-16T00:18:13,247 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/namespace/5982437be6e272746f8b60f107f87ef4/.tmp/info/7868105215364f7d8bc23915bcf92b4d is 45, key is default/info:d/1734308176047/Put/seqid=0
2024-12-16T00:18:13,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741876_1052 (size=5037)
2024-12-16T00:18:13,254 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741876_1052 (size=5037)
2024-12-16T00:18:13,254 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/namespace/5982437be6e272746f8b60f107f87ef4/.tmp/info/7868105215364f7d8bc23915bcf92b4d
2024-12-16T00:18:13,261 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/namespace/5982437be6e272746f8b60f107f87ef4/.tmp/info/7868105215364f7d8bc23915bcf92b4d as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/namespace/5982437be6e272746f8b60f107f87ef4/info/7868105215364f7d8bc23915bcf92b4d
2024-12-16T00:18:13,266 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/namespace/5982437be6e272746f8b60f107f87ef4/info/7868105215364f7d8bc23915bcf92b4d, entries=2, sequenceid=6, filesize=4.9 K
2024-12-16T00:18:13,267 INFO  [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 5982437be6e272746f8b60f107f87ef4 in 34ms, sequenceid=6, compaction requested=false
2024-12-16T00:18:13,267 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 5982437be6e272746f8b60f107f87ef4:

2024-12-16T00:18:13,267 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 9be904cf4ade01c9c8b0eed740afacf6:

2024-12-16T00:18:13,267 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 9b52be8ec131a4a57b605ce3472e9bf6 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-16T00:18:13,271 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/347ff34de17c4172b0aae35231973c35 is 1080, key is row0256/info:/1734308291177/Put/seqid=0
2024-12-16T00:18:13,276 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741877_1053 (size=6035)
2024-12-16T00:18:13,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741877_1053 (size=6035)
2024-12-16T00:18:13,642 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:13,678 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/347ff34de17c4172b0aae35231973c35
2024-12-16T00:18:13,684 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/.tmp/info/347ff34de17c4172b0aae35231973c35 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/347ff34de17c4172b0aae35231973c35
2024-12-16T00:18:13,690 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/347ff34de17c4172b0aae35231973c35, entries=1, sequenceid=329, filesize=5.9 K
2024-12-16T00:18:13,692 INFO  [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 9b52be8ec131a4a57b605ce3472e9bf6 in 424ms, sequenceid=329, compaction requested=false
2024-12-16T00:18:13,692 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:18:13,692 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C42285%2C1734308174608.1734308293692
2024-12-16T00:18:13,699 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608/b878af803003%2C42285%2C1734308174608.1734308293178 with entries=4, filesize=1.22 KB; new WAL /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608/b878af803003%2C42285%2C1734308174608.1734308293692
2024-12-16T00:18:13,699 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43035:43035),(127.0.0.1/127.0.0.1:37683:37683)]
2024-12-16T00:18:13,699 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608/b878af803003%2C42285%2C1734308174608.1734308293178 is not closed yet, will try archiving it next time
2024-12-16T00:18:13,699 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608/b878af803003%2C42285%2C1734308174608.1734308175117 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/oldWALs/b878af803003%2C42285%2C1734308174608.1734308175117
2024-12-16T00:18:13,700 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-12-16T00:18:13,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741874_1050 (size=1255)
2024-12-16T00:18:13,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741874_1050 (size=1255)
2024-12-16T00:18:13,702 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608/b878af803003%2C42285%2C1734308174608.1734308293178 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/oldWALs/b878af803003%2C42285%2C1734308174608.1734308293178
2024-12-16T00:18:13,800 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-16T00:18:13,800 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-16T00:18:13,800 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x13783cfc to 127.0.0.1:51475
2024-12-16T00:18:13,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:18:13,801 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-16T00:18:13,801 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1564858937, stopped=false
2024-12-16T00:18:13,801 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=b878af803003,44525,1734308174458
2024-12-16T00:18:13,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:18:13,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:18:13,840 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-16T00:18:13,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:13,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:13,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:18:13,840 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,42285,1734308174608' *****
2024-12-16T00:18:13,840 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-16T00:18:13,840 INFO  [RS:0;b878af803003:42285 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-16T00:18:13,840 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-16T00:18:13,840 INFO  [RS:0;b878af803003:42285 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-16T00:18:13,841 INFO  [RS:0;b878af803003:42285 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-16T00:18:13,841 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(3579): Received CLOSE for 5982437be6e272746f8b60f107f87ef4
2024-12-16T00:18:13,841 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:18:13,841 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:18:13,841 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(3579): Received CLOSE for 9be904cf4ade01c9c8b0eed740afacf6
2024-12-16T00:18:13,841 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(3579): Received CLOSE for 9b52be8ec131a4a57b605ce3472e9bf6
2024-12-16T00:18:13,841 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 5982437be6e272746f8b60f107f87ef4, disabling compactions & flushes
2024-12-16T00:18:13,841 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1224): stopping server b878af803003,42285,1734308174608
2024-12-16T00:18:13,841 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:18:13,841 DEBUG [RS:0;b878af803003:42285 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:18:13,841 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:18:13,841 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4. after waiting 0 ms
2024-12-16T00:18:13,841 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:18:13,841 INFO  [RS:0;b878af803003:42285 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-16T00:18:13,841 INFO  [RS:0;b878af803003:42285 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-16T00:18:13,841 INFO  [RS:0;b878af803003:42285 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-16T00:18:13,841 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-16T00:18:13,841 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1599): Waiting on 4 regions to close
2024-12-16T00:18:13,841 DEBUG [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, 5982437be6e272746f8b60f107f87ef4=hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4., 9be904cf4ade01c9c8b0eed740afacf6=TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6., 9b52be8ec131a4a57b605ce3472e9bf6=TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.}
2024-12-16T00:18:13,842 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:18:13,842 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:18:13,842 DEBUG [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 5982437be6e272746f8b60f107f87ef4, 9b52be8ec131a4a57b605ce3472e9bf6, 9be904cf4ade01c9c8b0eed740afacf6
2024-12-16T00:18:13,842 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:18:13,842 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:18:13,842 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:18:13,845 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/namespace/5982437be6e272746f8b60f107f87ef4/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-16T00:18:13,845 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/hbase/meta/1588230740/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1
2024-12-16T00:18:13,846 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:18:13,846 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 5982437be6e272746f8b60f107f87ef4:

2024-12-16T00:18:13,846 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1734308175597.5982437be6e272746f8b60f107f87ef4.
2024-12-16T00:18:13,846 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-16T00:18:13,846 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-16T00:18:13,846 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9be904cf4ade01c9c8b0eed740afacf6, disabling compactions & flushes
2024-12-16T00:18:13,846 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:18:13,846 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.
2024-12-16T00:18:13,846 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.
2024-12-16T00:18:13,846 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6. after waiting 0 ms
2024-12-16T00:18:13,846 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-16T00:18:13,846 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.
2024-12-16T00:18:13,846 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70->hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/53429a8b29a04f8c8a38fa2bdece6bfc-bottom] to archive
2024-12-16T00:18:13,847 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-16T00:18:13,849 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:18:13,853 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9be904cf4ade01c9c8b0eed740afacf6/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=121
2024-12-16T00:18:13,854 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.
2024-12-16T00:18:13,854 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9be904cf4ade01c9c8b0eed740afacf6:

2024-12-16T00:18:13,854 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1734308200522.9be904cf4ade01c9c8b0eed740afacf6.
2024-12-16T00:18:13,854 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9b52be8ec131a4a57b605ce3472e9bf6, disabling compactions & flushes
2024-12-16T00:18:13,854 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:18:13,854 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:18:13,854 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6. after waiting 0 ms
2024-12-16T00:18:13,854 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:18:13,855 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70->hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/f4d6c0aff037d3eb34d03175b6108e70/info/53429a8b29a04f8c8a38fa2bdece6bfc-top, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-474463326e624a91a8633a95e1dfd556, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/6d1243e026e94b1384cb46fafaf623fa, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-7946629a755740a7be6a1ef888d60c50, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/99185f634a654216b50d7ee8ac3febdd, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8a78f64ecffb4b5e84e2056bbf55de68, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/19e01ea6eadb49d1a33f5280b309efaf, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/622e457aae374ccaabd2abb3cd668b6d, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/9942f1db6b544068a26c8dc13646d9f0, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/fd0f6fc0e5df4b2fb530275ea0d8996f, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/e463f8502b2743cda9f0468bb54b5b23, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/2dfa4452a492435fb11110ad43394416, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8836841ff1424523b03b5a81f650f754, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/f033133eb95346ed99833dca626a1a97, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/f742e113e90047c5813f364ef37b56db, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/9e86af3537ae4f448298b4efc75ff073, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/13d5c7632e914d08a79f2a57e73cc198, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/bda99fef680942e49fbc945a07bea41d, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/317a9416f5cb4ec998d2ff95904a0c71, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/2b714407015f4df1a4473ecb1eb65550, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8de63d09f2b64a6f8af797067efc9fa6] to archive
2024-12-16T00:18:13,856 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-16T00:18:13,859 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/53429a8b29a04f8c8a38fa2bdece6bfc.f4d6c0aff037d3eb34d03175b6108e70
2024-12-16T00:18:13,859 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-474463326e624a91a8633a95e1dfd556 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-474463326e624a91a8633a95e1dfd556
2024-12-16T00:18:13,859 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/6d1243e026e94b1384cb46fafaf623fa to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/6d1243e026e94b1384cb46fafaf623fa
2024-12-16T00:18:13,859 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-7946629a755740a7be6a1ef888d60c50 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/TestLogRolling-testLogRolling=f4d6c0aff037d3eb34d03175b6108e70-7946629a755740a7be6a1ef888d60c50
2024-12-16T00:18:13,861 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/99185f634a654216b50d7ee8ac3febdd to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/99185f634a654216b50d7ee8ac3febdd
2024-12-16T00:18:13,861 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/9942f1db6b544068a26c8dc13646d9f0 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/9942f1db6b544068a26c8dc13646d9f0
2024-12-16T00:18:13,861 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8a78f64ecffb4b5e84e2056bbf55de68 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8a78f64ecffb4b5e84e2056bbf55de68
2024-12-16T00:18:13,862 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/19e01ea6eadb49d1a33f5280b309efaf to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/19e01ea6eadb49d1a33f5280b309efaf
2024-12-16T00:18:13,862 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/fd0f6fc0e5df4b2fb530275ea0d8996f to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/fd0f6fc0e5df4b2fb530275ea0d8996f
2024-12-16T00:18:13,862 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/622e457aae374ccaabd2abb3cd668b6d to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/622e457aae374ccaabd2abb3cd668b6d
2024-12-16T00:18:13,862 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/e463f8502b2743cda9f0468bb54b5b23 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/e463f8502b2743cda9f0468bb54b5b23
2024-12-16T00:18:13,862 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/2dfa4452a492435fb11110ad43394416 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/2dfa4452a492435fb11110ad43394416
2024-12-16T00:18:13,863 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8836841ff1424523b03b5a81f650f754 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8836841ff1424523b03b5a81f650f754
2024-12-16T00:18:13,864 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/f033133eb95346ed99833dca626a1a97 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/f033133eb95346ed99833dca626a1a97
2024-12-16T00:18:13,864 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/f742e113e90047c5813f364ef37b56db to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/f742e113e90047c5813f364ef37b56db
2024-12-16T00:18:13,864 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/9e86af3537ae4f448298b4efc75ff073 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/9e86af3537ae4f448298b4efc75ff073
2024-12-16T00:18:13,867 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/bda99fef680942e49fbc945a07bea41d to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/bda99fef680942e49fbc945a07bea41d
2024-12-16T00:18:13,867 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/13d5c7632e914d08a79f2a57e73cc198 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/13d5c7632e914d08a79f2a57e73cc198
2024-12-16T00:18:13,867 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/2b714407015f4df1a4473ecb1eb65550 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/2b714407015f4df1a4473ecb1eb65550
2024-12-16T00:18:13,867 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8de63d09f2b64a6f8af797067efc9fa6 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/8de63d09f2b64a6f8af797067efc9fa6
2024-12-16T00:18:13,867 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/317a9416f5cb4ec998d2ff95904a0c71 to hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/archive/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/info/317a9416f5cb4ec998d2ff95904a0c71
2024-12-16T00:18:13,873 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/data/default/TestLogRolling-testLogRolling/9b52be8ec131a4a57b605ce3472e9bf6/recovered.edits/332.seqid, newMaxSeqId=332, maxSeqId=121
2024-12-16T00:18:13,873 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:18:13,873 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9b52be8ec131a4a57b605ce3472e9bf6:

2024-12-16T00:18:13,873 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1734308200522.9b52be8ec131a4a57b605ce3472e9bf6.
2024-12-16T00:18:13,988 INFO  [regionserver/b878af803003:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-16T00:18:13,988 INFO  [regionserver/b878af803003:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-16T00:18:14,042 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1250): stopping server b878af803003,42285,1734308174608; all regions closed.
2024-12-16T00:18:14,042 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608
2024-12-16T00:18:14,044 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741834_1010 (size=9351)
2024-12-16T00:18:14,045 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741834_1010 (size=9351)
2024-12-16T00:18:14,048 DEBUG [RS:0;b878af803003:42285 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/oldWALs
2024-12-16T00:18:14,048 INFO  [RS:0;b878af803003:42285 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog b878af803003%2C42285%2C1734308174608.meta:.meta(num 1734308175492)
2024-12-16T00:18:14,049 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/WALs/b878af803003,42285,1734308174608
2024-12-16T00:18:14,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741878_1054 (size=1071)
2024-12-16T00:18:14,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741878_1054 (size=1071)
2024-12-16T00:18:14,055 DEBUG [RS:0;b878af803003:42285 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/oldWALs
2024-12-16T00:18:14,055 INFO  [RS:0;b878af803003:42285 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog b878af803003%2C42285%2C1734308174608:(num 1734308293692)
2024-12-16T00:18:14,055 DEBUG [RS:0;b878af803003:42285 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:18:14,055 INFO  [RS:0;b878af803003:42285 {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:18:14,055 INFO  [RS:0;b878af803003:42285 {}] hbase.ChoreService(370): Chore service for: regionserver/b878af803003:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown
2024-12-16T00:18:14,055 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:18:14,056 INFO  [RS:0;b878af803003:42285 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:42285
2024-12-16T00:18:14,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b878af803003,42285,1734308174608
2024-12-16T00:18:14,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:18:14,073 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b878af803003,42285,1734308174608]
2024-12-16T00:18:14,073 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing b878af803003,42285,1734308174608; numProcessing=1
2024-12-16T00:18:14,082 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/b878af803003,42285,1734308174608 already deleted, retry=false
2024-12-16T00:18:14,082 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; b878af803003,42285,1734308174608 expired; onlineServers=0
2024-12-16T00:18:14,082 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,44525,1734308174458' *****
2024-12-16T00:18:14,082 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-16T00:18:14,082 DEBUG [M:0;b878af803003:44525 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c97ff1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:18:14,082 INFO  [M:0;b878af803003:44525 {}] regionserver.HRegionServer(1224): stopping server b878af803003,44525,1734308174458
2024-12-16T00:18:14,082 INFO  [M:0;b878af803003:44525 {}] regionserver.HRegionServer(1250): stopping server b878af803003,44525,1734308174458; all regions closed.
2024-12-16T00:18:14,082 DEBUG [M:0;b878af803003:44525 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:18:14,083 DEBUG [M:0;b878af803003:44525 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-16T00:18:14,083 DEBUG [M:0;b878af803003:44525 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-16T00:18:14,083 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-16T00:18:14,083 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308174870 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308174870,5,FailOnTimeoutGroup]
2024-12-16T00:18:14,083 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308174871 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308174871,5,FailOnTimeoutGroup]
2024-12-16T00:18:14,083 INFO  [M:0;b878af803003:44525 {}] hbase.ChoreService(370): Chore service for: master/b878af803003:0 had [] on shutdown
2024-12-16T00:18:14,083 DEBUG [M:0;b878af803003:44525 {}] master.HMaster(1733): Stopping service threads
2024-12-16T00:18:14,084 INFO  [M:0;b878af803003:44525 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-16T00:18:14,084 INFO  [M:0;b878af803003:44525 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-16T00:18:14,084 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-16T00:18:14,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-16T00:18:14,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:14,148 DEBUG [M:0;b878af803003:44525 {}] zookeeper.ZKUtil(347): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-16T00:18:14,148 WARN  [M:0;b878af803003:44525 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-16T00:18:14,148 INFO  [M:0;b878af803003:44525 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-16T00:18:14,148 INFO  [M:0;b878af803003:44525 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-16T00:18:14,149 DEBUG [M:0;b878af803003:44525 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:18:14,149 INFO  [M:0;b878af803003:44525 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:14,149 DEBUG [M:0;b878af803003:44525 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:14,149 DEBUG [M:0;b878af803003:44525 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:18:14,149 DEBUG [M:0;b878af803003:44525 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:14,149 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:18:14,149 INFO  [M:0;b878af803003:44525 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=66.44 KB heapSize=81.69 KB
2024-12-16T00:18:14,167 DEBUG [M:0;b878af803003:44525 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ef0b0562106142e6b7d85c172fc7e03e is 82, key is hbase:meta,,1/info:regioninfo/1734308175514/Put/seqid=0
2024-12-16T00:18:14,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741879_1055 (size=5672)
2024-12-16T00:18:14,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741879_1055 (size=5672)
2024-12-16T00:18:14,172 INFO  [M:0;b878af803003:44525 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ef0b0562106142e6b7d85c172fc7e03e
2024-12-16T00:18:14,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:18:14,173 INFO  [RS:0;b878af803003:42285 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,42285,1734308174608; zookeeper connection closed.
2024-12-16T00:18:14,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42285-0x1002c31b6ee0001, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:18:14,173 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@617bc0eb {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@617bc0eb
2024-12-16T00:18:14,174 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-16T00:18:14,188 DEBUG [M:0;b878af803003:44525 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/617895e5cda348ab8ca1d0ee79a98c59 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1734308176683/Put/seqid=0
2024-12-16T00:18:14,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741880_1056 (size=7285)
2024-12-16T00:18:14,193 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741880_1056 (size=7285)
2024-12-16T00:18:14,193 INFO  [M:0;b878af803003:44525 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65.83 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/617895e5cda348ab8ca1d0ee79a98c59
2024-12-16T00:18:14,197 INFO  [M:0;b878af803003:44525 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 617895e5cda348ab8ca1d0ee79a98c59
2024-12-16T00:18:14,209 DEBUG [M:0;b878af803003:44525 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fe84ab6d1fb44ca0adbd19bc4be53b91 is 69, key is b878af803003,42285,1734308174608/rs:state/1734308174961/Put/seqid=0
2024-12-16T00:18:14,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741881_1057 (size=5156)
2024-12-16T00:18:14,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741881_1057 (size=5156)
2024-12-16T00:18:14,214 INFO  [M:0;b878af803003:44525 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fe84ab6d1fb44ca0adbd19bc4be53b91
2024-12-16T00:18:14,233 DEBUG [M:0;b878af803003:44525 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6229cb32500f401aaa4b25f0940f78fd is 52, key is load_balancer_on/state:d/1734308176246/Put/seqid=0
2024-12-16T00:18:14,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741882_1058 (size=5056)
2024-12-16T00:18:14,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741882_1058 (size=5056)
2024-12-16T00:18:14,238 INFO  [M:0;b878af803003:44525 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6229cb32500f401aaa4b25f0940f78fd
2024-12-16T00:18:14,243 DEBUG [M:0;b878af803003:44525 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ef0b0562106142e6b7d85c172fc7e03e as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ef0b0562106142e6b7d85c172fc7e03e
2024-12-16T00:18:14,246 INFO  [M:0;b878af803003:44525 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ef0b0562106142e6b7d85c172fc7e03e, entries=8, sequenceid=164, filesize=5.5 K
2024-12-16T00:18:14,247 DEBUG [M:0;b878af803003:44525 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/617895e5cda348ab8ca1d0ee79a98c59 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/617895e5cda348ab8ca1d0ee79a98c59
2024-12-16T00:18:14,250 INFO  [M:0;b878af803003:44525 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 617895e5cda348ab8ca1d0ee79a98c59
2024-12-16T00:18:14,250 INFO  [M:0;b878af803003:44525 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/617895e5cda348ab8ca1d0ee79a98c59, entries=18, sequenceid=164, filesize=7.1 K
2024-12-16T00:18:14,251 DEBUG [M:0;b878af803003:44525 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fe84ab6d1fb44ca0adbd19bc4be53b91 as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fe84ab6d1fb44ca0adbd19bc4be53b91
2024-12-16T00:18:14,255 INFO  [M:0;b878af803003:44525 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fe84ab6d1fb44ca0adbd19bc4be53b91, entries=1, sequenceid=164, filesize=5.0 K
2024-12-16T00:18:14,256 DEBUG [M:0;b878af803003:44525 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6229cb32500f401aaa4b25f0940f78fd as hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6229cb32500f401aaa4b25f0940f78fd
2024-12-16T00:18:14,260 INFO  [M:0;b878af803003:44525 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38027/user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6229cb32500f401aaa4b25f0940f78fd, entries=1, sequenceid=164, filesize=4.9 K
2024-12-16T00:18:14,261 INFO  [M:0;b878af803003:44525 {}] regionserver.HRegion(3040): Finished flush of dataSize ~66.44 KB/68031, heapSize ~81.63 KB/83584, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 112ms, sequenceid=164, compaction requested=false
2024-12-16T00:18:14,262 INFO  [M:0;b878af803003:44525 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:14,262 DEBUG [M:0;b878af803003:44525 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:18:14,263 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/2d3da449-ef20-b11b-ccd8-6c21c66e738e/MasterData/WALs/b878af803003,44525,1734308174458
2024-12-16T00:18:14,264 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39563 is added to blk_1073741830_1006 (size=79260)
2024-12-16T00:18:14,264 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35593 is added to blk_1073741830_1006 (size=79260)
2024-12-16T00:18:14,265 INFO  [M:0;b878af803003:44525 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-16T00:18:14,265 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:18:14,265 INFO  [M:0;b878af803003:44525 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:44525
2024-12-16T00:18:14,273 DEBUG [M:0;b878af803003:44525 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/b878af803003,44525,1734308174458 already deleted, retry=false
2024-12-16T00:18:14,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:18:14,382 INFO  [M:0;b878af803003:44525 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,44525,1734308174458; zookeeper connection closed.
2024-12-16T00:18:14,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44525-0x1002c31b6ee0000, quorum=127.0.0.1:51475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:18:14,384 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34dedf66{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:18:14,384 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@81c66a8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:18:14,384 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:18:14,384 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51ad3217{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:18:14,384 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70f0d727{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/hadoop.log.dir/,STOPPED}
2024-12-16T00:18:14,386 WARN  [BP-518996559-172.17.0.2-1734308172926 heartbeating to localhost/127.0.0.1:38027 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:18:14,386 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:18:14,386 WARN  [BP-518996559-172.17.0.2-1734308172926 heartbeating to localhost/127.0.0.1:38027 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-518996559-172.17.0.2-1734308172926 (Datanode Uuid 44782388-85e2-4e89-a04e-7cb03c5cff56) service to localhost/127.0.0.1:38027
2024-12-16T00:18:14,386 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:18:14,386 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9/dfs/data/data3/current/BP-518996559-172.17.0.2-1734308172926 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:18:14,386 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9/dfs/data/data4/current/BP-518996559-172.17.0.2-1734308172926 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:18:14,387 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:18:14,394 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@77ca63ce{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:18:14,394 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@39280d35{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:18:14,394 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:18:14,395 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b886661{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:18:14,395 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e5c2e9f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/hadoop.log.dir/,STOPPED}
2024-12-16T00:18:14,396 WARN  [BP-518996559-172.17.0.2-1734308172926 heartbeating to localhost/127.0.0.1:38027 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:18:14,396 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:18:14,396 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:18:14,396 WARN  [BP-518996559-172.17.0.2-1734308172926 heartbeating to localhost/127.0.0.1:38027 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-518996559-172.17.0.2-1734308172926 (Datanode Uuid a4bd9524-63e4-4198-9cef-1a4439074e39) service to localhost/127.0.0.1:38027
2024-12-16T00:18:14,397 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9/dfs/data/data1/current/BP-518996559-172.17.0.2-1734308172926 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:18:14,397 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/cluster_6b1cbef5-5543-055d-1296-5cf975a3cdc9/dfs/data/data2/current/BP-518996559-172.17.0.2-1734308172926 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:18:14,397 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:18:14,403 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@d34012c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:18:14,404 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@68bfa336{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:18:14,404 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:18:14,404 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@691fae27{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:18:14,404 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a90e359{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/hadoop.log.dir/,STOPPED}
2024-12-16T00:18:14,412 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-16T00:18:14,451 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-16T00:18:14,458 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=130 (was 111) - Thread LEAK? -, OpenFileDescriptor=490 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=47 (was 40) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12133 (was 11904) - AvailableMemoryMB LEAK? -
2024-12-16T00:18:14,465 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=131, OpenFileDescriptor=490, MaxFileDescriptor=1048576, SystemLoadAverage=47, ProcessCount=11, AvailableMemoryMB=12133
2024-12-16T00:18:14,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-16T00:18:14,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/hadoop.log.dir so I do NOT create it in target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7
2024-12-16T00:18:14,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/f6b90f09-b8d4-10f7-5b0b-9fe4cb403bd4/hadoop.tmp.dir so I do NOT create it in target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7
2024-12-16T00:18:14,465 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb, deleteOnExit=true
2024-12-16T00:18:14,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-16T00:18:14,465 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/test.cache.data in system properties and HBase conf
2024-12-16T00:18:14,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/hadoop.tmp.dir in system properties and HBase conf
2024-12-16T00:18:14,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/hadoop.log.dir in system properties and HBase conf
2024-12-16T00:18:14,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-16T00:18:14,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-16T00:18:14,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-16T00:18:14,466 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-16T00:18:14,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:18:14,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-16T00:18:14,466 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-16T00:18:14,467 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:18:14,467 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-16T00:18:14,467 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-16T00:18:14,467 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-16T00:18:14,467 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:18:14,467 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-16T00:18:14,467 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/nfs.dump.dir in system properties and HBase conf
2024-12-16T00:18:14,467 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/java.io.tmpdir in system properties and HBase conf
2024-12-16T00:18:14,467 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-16T00:18:14,468 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-16T00:18:14,468 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-16T00:18:14,483 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:18:14,642 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:14,789 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:18:14,792 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:18:14,793 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:18:14,793 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:18:14,793 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:18:14,793 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:18:14,794 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@166b3543{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:18:14,794 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10765324{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:18:14,883 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d2c54de{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/java.io.tmpdir/jetty-localhost-44293-hadoop-hdfs-3_4_1-tests_jar-_-any-12277903980245356669/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:18:14,883 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6ee33572{HTTP/1.1, (http/1.1)}{localhost:44293}
2024-12-16T00:18:14,883 INFO  [Time-limited test {}] server.Server(415): Started @413548ms
2024-12-16T00:18:14,893 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-16T00:18:14,991 INFO  [regionserver/b878af803003:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:18:15,105 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:18:15,107 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:18:15,109 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:18:15,109 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:18:15,109 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:18:15,110 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29d9a359{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:18:15,110 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1328adb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:18:15,198 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5226d17{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/java.io.tmpdir/jetty-localhost-37039-hadoop-hdfs-3_4_1-tests_jar-_-any-13122938742447291856/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:18:15,198 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@33561086{HTTP/1.1, (http/1.1)}{localhost:37039}
2024-12-16T00:18:15,199 INFO  [Time-limited test {}] server.Server(415): Started @413863ms
2024-12-16T00:18:15,200 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:18:15,224 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-16T00:18:15,226 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-16T00:18:15,227 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-16T00:18:15,227 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-16T00:18:15,227 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-16T00:18:15,227 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34c1e0e0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/hadoop.log.dir/,AVAILABLE}
2024-12-16T00:18:15,228 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@421d8f5d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-16T00:18:15,326 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6573ca10{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/java.io.tmpdir/jetty-localhost-45735-hadoop-hdfs-3_4_1-tests_jar-_-any-17793081200702689240/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:18:15,326 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@158dc7e4{HTTP/1.1, (http/1.1)}{localhost:45735}
2024-12-16T00:18:15,326 INFO  [Time-limited test {}] server.Server(415): Started @413991ms
2024-12-16T00:18:15,327 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-16T00:18:15,643 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:15,793 WARN  [Thread-2265 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb/dfs/data/data2/current/BP-1183266520-172.17.0.2-1734308294502/current, will proceed with Du for space computation calculation, 
2024-12-16T00:18:15,793 WARN  [Thread-2264 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb/dfs/data/data1/current/BP-1183266520-172.17.0.2-1734308294502/current, will proceed with Du for space computation calculation, 
2024-12-16T00:18:15,807 WARN  [Thread-2229 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:18:15,809 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x47fad6e1ed580bd5 with lease ID 0xc79393e19ecccc11: Processing first storage report for DS-57e22206-098f-44d9-8307-3121999eed8f from datanode DatanodeRegistration(127.0.0.1:36727, datanodeUuid=931090f2-334a-49a1-a6b7-c5ab031fb8ce, infoPort=46501, infoSecurePort=0, ipcPort=41469, storageInfo=lv=-57;cid=testClusterID;nsid=731638877;c=1734308294502)
2024-12-16T00:18:15,809 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x47fad6e1ed580bd5 with lease ID 0xc79393e19ecccc11: from storage DS-57e22206-098f-44d9-8307-3121999eed8f node DatanodeRegistration(127.0.0.1:36727, datanodeUuid=931090f2-334a-49a1-a6b7-c5ab031fb8ce, infoPort=46501, infoSecurePort=0, ipcPort=41469, storageInfo=lv=-57;cid=testClusterID;nsid=731638877;c=1734308294502), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:18:15,809 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x47fad6e1ed580bd5 with lease ID 0xc79393e19ecccc11: Processing first storage report for DS-3d4f5c81-2f20-4e42-843e-07c0e6d49c30 from datanode DatanodeRegistration(127.0.0.1:36727, datanodeUuid=931090f2-334a-49a1-a6b7-c5ab031fb8ce, infoPort=46501, infoSecurePort=0, ipcPort=41469, storageInfo=lv=-57;cid=testClusterID;nsid=731638877;c=1734308294502)
2024-12-16T00:18:15,809 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x47fad6e1ed580bd5 with lease ID 0xc79393e19ecccc11: from storage DS-3d4f5c81-2f20-4e42-843e-07c0e6d49c30 node DatanodeRegistration(127.0.0.1:36727, datanodeUuid=931090f2-334a-49a1-a6b7-c5ab031fb8ce, infoPort=46501, infoSecurePort=0, ipcPort=41469, storageInfo=lv=-57;cid=testClusterID;nsid=731638877;c=1734308294502), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:18:15,960 WARN  [Thread-2276 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb/dfs/data/data3/current/BP-1183266520-172.17.0.2-1734308294502/current, will proceed with Du for space computation calculation, 
2024-12-16T00:18:15,960 WARN  [Thread-2277 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb/dfs/data/data4/current/BP-1183266520-172.17.0.2-1734308294502/current, will proceed with Du for space computation calculation, 
2024-12-16T00:18:15,978 WARN  [Thread-2252 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-16T00:18:15,980 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x11a8af3cade87848 with lease ID 0xc79393e19ecccc12: Processing first storage report for DS-d4760d46-e0ef-4215-ab1d-c493ceb6c3b9 from datanode DatanodeRegistration(127.0.0.1:38433, datanodeUuid=af7fd491-c695-4eb4-8adf-33848a4d4a98, infoPort=33271, infoSecurePort=0, ipcPort=39519, storageInfo=lv=-57;cid=testClusterID;nsid=731638877;c=1734308294502)
2024-12-16T00:18:15,980 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11a8af3cade87848 with lease ID 0xc79393e19ecccc12: from storage DS-d4760d46-e0ef-4215-ab1d-c493ceb6c3b9 node DatanodeRegistration(127.0.0.1:38433, datanodeUuid=af7fd491-c695-4eb4-8adf-33848a4d4a98, infoPort=33271, infoSecurePort=0, ipcPort=39519, storageInfo=lv=-57;cid=testClusterID;nsid=731638877;c=1734308294502), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:18:15,980 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x11a8af3cade87848 with lease ID 0xc79393e19ecccc12: Processing first storage report for DS-093a5cc1-c623-4027-9865-4cc444ec60ed from datanode DatanodeRegistration(127.0.0.1:38433, datanodeUuid=af7fd491-c695-4eb4-8adf-33848a4d4a98, infoPort=33271, infoSecurePort=0, ipcPort=39519, storageInfo=lv=-57;cid=testClusterID;nsid=731638877;c=1734308294502)
2024-12-16T00:18:15,980 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11a8af3cade87848 with lease ID 0xc79393e19ecccc12: from storage DS-093a5cc1-c623-4027-9865-4cc444ec60ed node DatanodeRegistration(127.0.0.1:38433, datanodeUuid=af7fd491-c695-4eb4-8adf-33848a4d4a98, infoPort=33271, infoSecurePort=0, ipcPort=39519, storageInfo=lv=-57;cid=testClusterID;nsid=731638877;c=1734308294502), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-16T00:18:16,052 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7
2024-12-16T00:18:16,057 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb/zookeeper_0, clientPort=58426, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-16T00:18:16,058 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=58426
2024-12-16T00:18:16,058 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:18:16,060 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:18:16,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:18:16,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741825_1001 (size=7)
2024-12-16T00:18:16,071 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c with version=8
2024-12-16T00:18:16,071 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:33445/user/jenkins/test-data/3bb74fd5-9093-6218-8cf3-86b2c2d89d17/hbase-staging
2024-12-16T00:18:16,073 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/b878af803003:0 server-side Connection retries=45
2024-12-16T00:18:16,073 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:18:16,073 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:18:16,073 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:18:16,073 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:18:16,073 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:18:16,073 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:18:16,073 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:18:16,074 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:42441
2024-12-16T00:18:16,074 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:18:16,076 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:18:16,077 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:42441 connecting to ZooKeeper ensemble=127.0.0.1:58426
2024-12-16T00:18:16,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:424410x0, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:18:16,135 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42441-0x1002c3391f70000 connected
2024-12-16T00:18:16,198 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:18:16,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:18:16,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:18:16,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42441
2024-12-16T00:18:16,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42441
2024-12-16T00:18:16,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42441
2024-12-16T00:18:16,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42441
2024-12-16T00:18:16,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42441
2024-12-16T00:18:16,201 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c, hbase.cluster.distributed=false
2024-12-16T00:18:16,216 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/b878af803003:0 server-side Connection retries=45
2024-12-16T00:18:16,216 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:18:16,216 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-16T00:18:16,216 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-16T00:18:16,216 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-16T00:18:16,216 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-16T00:18:16,216 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-16T00:18:16,216 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-16T00:18:16,217 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:35041
2024-12-16T00:18:16,217 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-16T00:18:16,218 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-16T00:18:16,219 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:18:16,220 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:18:16,222 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:35041 connecting to ZooKeeper ensemble=127.0.0.1:58426
2024-12-16T00:18:16,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350410x0, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-16T00:18:16,232 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35041-0x1002c3391f70001 connected
2024-12-16T00:18:16,232 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:18:16,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:18:16,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-16T00:18:16,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35041
2024-12-16T00:18:16,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35041
2024-12-16T00:18:16,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35041
2024-12-16T00:18:16,235 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35041
2024-12-16T00:18:16,235 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35041
2024-12-16T00:18:16,236 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/b878af803003,42441,1734308296072
2024-12-16T00:18:16,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:18:16,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:18:16,246 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b878af803003,42441,1734308296072
2024-12-16T00:18:16,249 DEBUG [M:0;b878af803003:42441 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b878af803003:42441
2024-12-16T00:18:16,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:18:16,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-16T00:18:16,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,256 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:18:16,257 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b878af803003,42441,1734308296072 from backup master directory
2024-12-16T00:18:16,257 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-16T00:18:16,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:18:16,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b878af803003,42441,1734308296072
2024-12-16T00:18:16,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-16T00:18:16,265 WARN  [master/b878af803003:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:18:16,265 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b878af803003,42441,1734308296072
2024-12-16T00:18:16,276 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:18:16,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741826_1002 (size=42)
2024-12-16T00:18:16,277 DEBUG [master/b878af803003:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/hbase.id with ID: bcd90808-14bd-4f03-905f-88c051144e64
2024-12-16T00:18:16,287 INFO  [master/b878af803003:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:18:16,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,303 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:18:16,303 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741827_1003 (size=196)
2024-12-16T00:18:16,304 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-16T00:18:16,305 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-16T00:18:16,305 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:18:16,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:18:16,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741828_1004 (size=1189)
2024-12-16T00:18:16,315 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store
2024-12-16T00:18:16,320 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:18:16,320 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741829_1005 (size=34)
2024-12-16T00:18:16,322 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:18:16,322 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:18:16,322 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:16,322 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:16,322 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:18:16,322 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:16,322 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:16,322 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:18:16,323 WARN  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/.initializing
2024-12-16T00:18:16,323 DEBUG [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/WALs/b878af803003,42441,1734308296072
2024-12-16T00:18:16,326 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C42441%2C1734308296072, suffix=, logDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/WALs/b878af803003,42441,1734308296072, archiveDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/oldWALs, maxLogs=10
2024-12-16T00:18:16,326 INFO  [master/b878af803003:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C42441%2C1734308296072.1734308296326
2024-12-16T00:18:16,332 INFO  [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/WALs/b878af803003,42441,1734308296072/b878af803003%2C42441%2C1734308296072.1734308296326
2024-12-16T00:18:16,332 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46501:46501),(127.0.0.1/127.0.0.1:33271:33271)]
2024-12-16T00:18:16,332 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:18:16,332 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:18:16,332 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:18:16,332 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:18:16,333 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:18:16,334 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-16T00:18:16,335 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:16,335 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:18:16,335 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:18:16,336 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-16T00:18:16,336 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:16,336 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:18:16,336 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:18:16,337 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-16T00:18:16,337 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:16,338 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:18:16,338 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-16T00:18:16,339 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-16T00:18:16,339 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:16,339 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:18:16,340 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:18:16,340 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:18:16,341 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-16T00:18:16,342 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-16T00:18:16,344 DEBUG [master/b878af803003:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:18:16,345 INFO  [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=706434, jitterRate=-0.10172370076179504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-16T00:18:16,345 DEBUG [master/b878af803003:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:18:16,345 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-16T00:18:16,348 DEBUG [master/b878af803003:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ed1a70f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:18:16,348 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-16T00:18:16,349 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-16T00:18:16,349 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-16T00:18:16,349 INFO  [master/b878af803003:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-16T00:18:16,349 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-16T00:18:16,349 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-16T00:18:16,349 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-16T00:18:16,351 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-16T00:18:16,352 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-16T00:18:16,356 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-16T00:18:16,356 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-16T00:18:16,357 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-16T00:18:16,365 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-16T00:18:16,365 INFO  [master/b878af803003:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-16T00:18:16,366 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-16T00:18:16,373 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-16T00:18:16,374 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-16T00:18:16,381 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-16T00:18:16,383 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-16T00:18:16,389 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-16T00:18:16,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:18:16,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-16T00:18:16,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,398 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=b878af803003,42441,1734308296072, sessionid=0x1002c3391f70000, setting cluster-up flag (Was=false)
2024-12-16T00:18:16,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,440 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-16T00:18:16,440 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,42441,1734308296072
2024-12-16T00:18:16,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,481 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-16T00:18:16,482 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b878af803003,42441,1734308296072
2024-12-16T00:18:16,485 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-16T00:18:16,485 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-16T00:18:16,485 INFO  [master/b878af803003:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-16T00:18:16,485 DEBUG [master/b878af803003:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b878af803003,42441,1734308296072
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-16T00:18:16,485 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:18:16,485 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:18:16,485 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:18:16,485 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b878af803003:0, corePoolSize=5, maxPoolSize=5
2024-12-16T00:18:16,485 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b878af803003:0, corePoolSize=10, maxPoolSize=10
2024-12-16T00:18:16,485 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,485 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:18:16,485 DEBUG [master/b878af803003:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,486 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:18:16,487 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-16T00:18:16,488 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:16,488 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:18:16,490 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734308326490
2024-12-16T00:18:16,490 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-16T00:18:16,490 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-16T00:18:16,490 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-16T00:18:16,490 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-16T00:18:16,490 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-16T00:18:16,490 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-16T00:18:16,490 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:16,496 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-16T00:18:16,496 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-16T00:18:16,496 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-16T00:18:16,497 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-16T00:18:16,497 INFO  [master/b878af803003:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-16T00:18:16,497 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308296497,5,FailOnTimeoutGroup]
2024-12-16T00:18:16,497 DEBUG [master/b878af803003:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308296497,5,FailOnTimeoutGroup]
2024-12-16T00:18:16,497 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:16,497 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-16T00:18:16,497 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:16,497 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:16,500 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:18:16,500 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741831_1007 (size=1039)
2024-12-16T00:18:16,501 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-16T00:18:16,501 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c
2024-12-16T00:18:16,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:18:16,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741832_1008 (size=32)
2024-12-16T00:18:16,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:18:16,511 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:18:16,512 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:18:16,512 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:16,513 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:18:16,513 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:18:16,514 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:18:16,514 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:16,514 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:18:16,514 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:18:16,515 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:18:16,515 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:16,515 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:18:16,516 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740
2024-12-16T00:18:16,516 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740
2024-12-16T00:18:16,517 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:18:16,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:18:16,519 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:18:16,519 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=743418, jitterRate=-0.05469638109207153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:18:16,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:18:16,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:18:16,519 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:18:16,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:18:16,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:18:16,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:18:16,520 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-16T00:18:16,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:18:16,520 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-16T00:18:16,520 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-16T00:18:16,521 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-16T00:18:16,521 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-16T00:18:16,522 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-16T00:18:16,549 DEBUG [RS:0;b878af803003:35041 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b878af803003:35041
2024-12-16T00:18:16,550 INFO  [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1008): ClusterId : bcd90808-14bd-4f03-905f-88c051144e64
2024-12-16T00:18:16,550 DEBUG [RS:0;b878af803003:35041 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-16T00:18:16,557 DEBUG [RS:0;b878af803003:35041 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-16T00:18:16,557 DEBUG [RS:0;b878af803003:35041 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-16T00:18:16,566 DEBUG [RS:0;b878af803003:35041 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-16T00:18:16,566 DEBUG [RS:0;b878af803003:35041 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ab71f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:18:16,566 DEBUG [RS:0;b878af803003:35041 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159d8e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:18:16,566 INFO  [RS:0;b878af803003:35041 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-16T00:18:16,566 INFO  [RS:0;b878af803003:35041 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-16T00:18:16,566 DEBUG [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-16T00:18:16,567 INFO  [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(3073): reportForDuty to master=b878af803003,42441,1734308296072 with isa=b878af803003/172.17.0.2:35041, startcode=1734308296215
2024-12-16T00:18:16,567 DEBUG [RS:0;b878af803003:35041 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-16T00:18:16,570 INFO  [RS-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56623, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-12-16T00:18:16,570 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42441 {}] master.ServerManager(332): Checking decommissioned status of RegionServer b878af803003,35041,1734308296215
2024-12-16T00:18:16,570 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42441 {}] master.ServerManager(486): Registering regionserver=b878af803003,35041,1734308296215
2024-12-16T00:18:16,572 DEBUG [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c
2024-12-16T00:18:16,572 DEBUG [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:43035
2024-12-16T00:18:16,572 DEBUG [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-16T00:18:16,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:18:16,582 DEBUG [RS:0;b878af803003:35041 {}] zookeeper.ZKUtil(111): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b878af803003,35041,1734308296215
2024-12-16T00:18:16,582 WARN  [RS:0;b878af803003:35041 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-16T00:18:16,582 INFO  [RS:0;b878af803003:35041 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:18:16,583 DEBUG [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/b878af803003,35041,1734308296215
2024-12-16T00:18:16,583 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b878af803003,35041,1734308296215]
2024-12-16T00:18:16,588 DEBUG [RS:0;b878af803003:35041 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-16T00:18:16,588 INFO  [RS:0;b878af803003:35041 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-16T00:18:16,591 INFO  [RS:0;b878af803003:35041 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-16T00:18:16,591 INFO  [RS:0;b878af803003:35041 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-16T00:18:16,591 INFO  [RS:0;b878af803003:35041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:16,591 INFO  [RS:0;b878af803003:35041 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-16T00:18:16,592 INFO  [RS:0;b878af803003:35041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:16,592 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,592 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,592 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,592 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,593 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,593 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b878af803003:0, corePoolSize=2, maxPoolSize=2
2024-12-16T00:18:16,593 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,593 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,593 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,593 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,593 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b878af803003:0, corePoolSize=1, maxPoolSize=1
2024-12-16T00:18:16,593 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:18:16,593 DEBUG [RS:0;b878af803003:35041 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b878af803003:0, corePoolSize=3, maxPoolSize=3
2024-12-16T00:18:16,593 INFO  [RS:0;b878af803003:35041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:16,594 INFO  [RS:0;b878af803003:35041 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:16,594 INFO  [RS:0;b878af803003:35041 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:16,594 INFO  [RS:0;b878af803003:35041 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:16,594 INFO  [RS:0;b878af803003:35041 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,35041,1734308296215-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:18:16,606 INFO  [RS:0;b878af803003:35041 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-16T00:18:16,606 INFO  [RS:0;b878af803003:35041 {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,35041,1734308296215-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:16,617 INFO  [RS:0;b878af803003:35041 {}] regionserver.Replication(204): b878af803003,35041,1734308296215 started
2024-12-16T00:18:16,617 INFO  [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1767): Serving as b878af803003,35041,1734308296215, RpcServer on b878af803003/172.17.0.2:35041, sessionid=0x1002c3391f70001
2024-12-16T00:18:16,617 DEBUG [RS:0;b878af803003:35041 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-16T00:18:16,617 DEBUG [RS:0;b878af803003:35041 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b878af803003,35041,1734308296215
2024-12-16T00:18:16,617 DEBUG [RS:0;b878af803003:35041 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,35041,1734308296215'
2024-12-16T00:18:16,617 DEBUG [RS:0;b878af803003:35041 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-16T00:18:16,618 DEBUG [RS:0;b878af803003:35041 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-16T00:18:16,618 DEBUG [RS:0;b878af803003:35041 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-16T00:18:16,618 DEBUG [RS:0;b878af803003:35041 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-16T00:18:16,618 DEBUG [RS:0;b878af803003:35041 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b878af803003,35041,1734308296215
2024-12-16T00:18:16,618 DEBUG [RS:0;b878af803003:35041 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b878af803003,35041,1734308296215'
2024-12-16T00:18:16,618 DEBUG [RS:0;b878af803003:35041 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-16T00:18:16,618 DEBUG [RS:0;b878af803003:35041 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-16T00:18:16,619 DEBUG [RS:0;b878af803003:35041 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-16T00:18:16,619 INFO  [RS:0;b878af803003:35041 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-16T00:18:16,619 INFO  [RS:0;b878af803003:35041 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-16T00:18:16,644 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:16,672 WARN  [b878af803003:42441 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-16T00:18:16,721 INFO  [RS:0;b878af803003:35041 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C35041%2C1734308296215, suffix=, logDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/b878af803003,35041,1734308296215, archiveDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/oldWALs, maxLogs=32
2024-12-16T00:18:16,722 INFO  [RS:0;b878af803003:35041 {}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C35041%2C1734308296215.1734308296721
2024-12-16T00:18:16,728 INFO  [RS:0;b878af803003:35041 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/b878af803003,35041,1734308296215/b878af803003%2C35041%2C1734308296215.1734308296721
2024-12-16T00:18:16,729 DEBUG [RS:0;b878af803003:35041 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46501:46501),(127.0.0.1/127.0.0.1:33271:33271)]
2024-12-16T00:18:16,862 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-16T00:18:16,922 DEBUG [b878af803003:42441 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-16T00:18:16,922 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b878af803003,35041,1734308296215
2024-12-16T00:18:16,924 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,35041,1734308296215, state=OPENING
2024-12-16T00:18:16,931 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-16T00:18:16,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:16,941 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=b878af803003,35041,1734308296215}]
2024-12-16T00:18:16,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:18:16,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:18:17,094 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to b878af803003,35041,1734308296215
2024-12-16T00:18:17,095 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-16T00:18:17,098 INFO  [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43102, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-16T00:18:17,101 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-16T00:18:17,101 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:18:17,103 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b878af803003%2C35041%2C1734308296215.meta, suffix=.meta, logDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/b878af803003,35041,1734308296215, archiveDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/oldWALs, maxLogs=32
2024-12-16T00:18:17,103 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor b878af803003%2C35041%2C1734308296215.meta.1734308297103.meta
2024-12-16T00:18:17,116 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/b878af803003,35041,1734308296215/b878af803003%2C35041%2C1734308296215.meta.1734308297103.meta
2024-12-16T00:18:17,116 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33271:33271),(127.0.0.1/127.0.0.1:46501:46501)]
2024-12-16T00:18:17,116 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:18:17,117 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-16T00:18:17,117 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-16T00:18:17,117 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-16T00:18:17,117 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-16T00:18:17,117 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:18:17,117 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-16T00:18:17,117 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-16T00:18:17,119 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-16T00:18:17,120 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-16T00:18:17,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:17,120 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:18:17,120 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-16T00:18:17,121 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-16T00:18:17,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:17,121 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:18:17,121 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-16T00:18:17,122 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-16T00:18:17,122 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:17,122 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-16T00:18:17,123 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740
2024-12-16T00:18:17,124 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740
2024-12-16T00:18:17,125 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-16T00:18:17,125 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-16T00:18:17,126 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=789624, jitterRate=0.004059255123138428}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-16T00:18:17,126 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-16T00:18:17,127 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1734308297094
2024-12-16T00:18:17,128 DEBUG [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-16T00:18:17,128 INFO  [RS_OPEN_META-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-16T00:18:17,128 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,35041,1734308296215
2024-12-16T00:18:17,129 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b878af803003,35041,1734308296215, state=OPEN
2024-12-16T00:18:17,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:18:17,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-16T00:18:17,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:18:17,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-16T00:18:17,187 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-16T00:18:17,187 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=b878af803003,35041,1734308296215 in 244 msec
2024-12-16T00:18:17,189 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-16T00:18:17,190 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 667 msec
2024-12-16T00:18:17,193 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 707 msec
2024-12-16T00:18:17,193 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1734308297193, completionTime=-1
2024-12-16T00:18:17,193 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-16T00:18:17,193 DEBUG [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-16T00:18:17,194 DEBUG [hconnection-0x11321e3f-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:18:17,195 INFO  [RS-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43116, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:18:17,196 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-16T00:18:17,196 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1734308357196
2024-12-16T00:18:17,196 INFO  [master/b878af803003:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1734308417196
2024-12-16T00:18:17,196 INFO  [master/b878af803003:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec
2024-12-16T00:18:17,223 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42441,1734308296072-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:17,223 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42441,1734308296072-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:17,223 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42441,1734308296072-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:17,223 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b878af803003:42441, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:17,223 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:17,224 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-16T00:18:17,224 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-16T00:18:17,225 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-16T00:18:17,225 DEBUG [master/b878af803003:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-16T00:18:17,226 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-16T00:18:17,226 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:17,226 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-16T00:18:17,232 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:18:17,232 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741835_1011 (size=358)
2024-12-16T00:18:17,234 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ca171cc0fba2d9cba9a8626247a8c647, NAME => 'hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c
2024-12-16T00:18:17,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:18:17,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741836_1012 (size=42)
2024-12-16T00:18:17,239 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:18:17,239 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing ca171cc0fba2d9cba9a8626247a8c647, disabling compactions & flushes
2024-12-16T00:18:17,239 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,239 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,239 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647. after waiting 0 ms
2024-12-16T00:18:17,239 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,239 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,239 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for ca171cc0fba2d9cba9a8626247a8c647:

2024-12-16T00:18:17,240 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-16T00:18:17,241 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1734308297240"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1734308297240"}]},"ts":"1734308297240"}
2024-12-16T00:18:17,242 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-16T00:18:17,243 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-16T00:18:17,243 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308297243"}]},"ts":"1734308297243"}
2024-12-16T00:18:17,244 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-16T00:18:17,265 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=ca171cc0fba2d9cba9a8626247a8c647, ASSIGN}]
2024-12-16T00:18:17,266 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=ca171cc0fba2d9cba9a8626247a8c647, ASSIGN
2024-12-16T00:18:17,267 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=ca171cc0fba2d9cba9a8626247a8c647, ASSIGN; state=OFFLINE, location=b878af803003,35041,1734308296215; forceNewPlan=false, retain=false
2024-12-16T00:18:17,417 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=ca171cc0fba2d9cba9a8626247a8c647, regionState=OPENING, regionLocation=b878af803003,35041,1734308296215
2024-12-16T00:18:17,419 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure ca171cc0fba2d9cba9a8626247a8c647, server=b878af803003,35041,1734308296215}]
2024-12-16T00:18:17,572 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to b878af803003,35041,1734308296215
2024-12-16T00:18:17,579 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => ca171cc0fba2d9cba9a8626247a8c647, NAME => 'hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.', STARTKEY => '', ENDKEY => ''}
2024-12-16T00:18:17,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace ca171cc0fba2d9cba9a8626247a8c647
2024-12-16T00:18:17,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-16T00:18:17,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for ca171cc0fba2d9cba9a8626247a8c647
2024-12-16T00:18:17,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for ca171cc0fba2d9cba9a8626247a8c647
2024-12-16T00:18:17,582 INFO  [StoreOpener-ca171cc0fba2d9cba9a8626247a8c647-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ca171cc0fba2d9cba9a8626247a8c647 
2024-12-16T00:18:17,584 INFO  [StoreOpener-ca171cc0fba2d9cba9a8626247a8c647-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca171cc0fba2d9cba9a8626247a8c647 columnFamilyName info
2024-12-16T00:18:17,584 DEBUG [StoreOpener-ca171cc0fba2d9cba9a8626247a8c647-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-16T00:18:17,585 INFO  [StoreOpener-ca171cc0fba2d9cba9a8626247a8c647-1 {}] regionserver.HStore(327): Store=ca171cc0fba2d9cba9a8626247a8c647/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-16T00:18:17,586 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/namespace/ca171cc0fba2d9cba9a8626247a8c647
2024-12-16T00:18:17,586 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/namespace/ca171cc0fba2d9cba9a8626247a8c647
2024-12-16T00:18:17,589 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for ca171cc0fba2d9cba9a8626247a8c647
2024-12-16T00:18:17,592 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/namespace/ca171cc0fba2d9cba9a8626247a8c647/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-16T00:18:17,592 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened ca171cc0fba2d9cba9a8626247a8c647; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=725555, jitterRate=-0.07741020619869232}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-16T00:18:17,592 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for ca171cc0fba2d9cba9a8626247a8c647:

2024-12-16T00:18:17,593 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647., pid=6, masterSystemTime=1734308297572
2024-12-16T00:18:17,594 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,594 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,595 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=ca171cc0fba2d9cba9a8626247a8c647, regionState=OPEN, openSeqNum=2, regionLocation=b878af803003,35041,1734308296215
2024-12-16T00:18:17,598 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-16T00:18:17,598 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure ca171cc0fba2d9cba9a8626247a8c647, server=b878af803003,35041,1734308296215 in 177 msec
2024-12-16T00:18:17,600 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-16T00:18:17,600 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=ca171cc0fba2d9cba9a8626247a8c647, ASSIGN in 333 msec
2024-12-16T00:18:17,600 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-16T00:18:17,601 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1734308297600"}]},"ts":"1734308297600"}
2024-12-16T00:18:17,602 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-16T00:18:17,640 DEBUG [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-16T00:18:17,641 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-16T00:18:17,642 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 417 msec
2024-12-16T00:18:17,644 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37443/user/jenkins/test-data/17109543-88fa-5d52-1ef3-af80b06133db/WALs/b878af803003,39441,1734308000099/b878af803003%2C39441%2C1734308000099.meta.1734308001078.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-16T00:18:17,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:18:17,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:17,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:17,652 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-16T00:18:17,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:18:17,675 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 23 msec
2024-12-16T00:18:17,684 DEBUG [master/b878af803003:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-16T00:18:17,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-16T00:18:17,708 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 24 msec
2024-12-16T00:18:17,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-16T00:18:17,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-16T00:18:17,748 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.483sec
2024-12-16T00:18:17,748 INFO  [master/b878af803003:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-16T00:18:17,748 INFO  [master/b878af803003:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-16T00:18:17,748 INFO  [master/b878af803003:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-16T00:18:17,748 INFO  [master/b878af803003:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-16T00:18:17,748 INFO  [master/b878af803003:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-16T00:18:17,748 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42441,1734308296072-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-16T00:18:17,748 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42441,1734308296072-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-16T00:18:17,750 DEBUG [master/b878af803003:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-16T00:18:17,750 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-16T00:18:17,750 INFO  [master/b878af803003:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b878af803003,42441,1734308296072-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-16T00:18:17,839 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5e7cfd0e to 127.0.0.1:58426 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ed0020a
2024-12-16T00:18:17,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@104f6f89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-16T00:18:17,853 DEBUG [hconnection-0x65ab69a0-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-16T00:18:17,855 INFO  [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43126, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-16T00:18:17,858 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=b878af803003,42441,1734308296072
2024-12-16T00:18:17,859 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-16T00:18:17,861 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-16T00:18:17,862 INFO  [Time-limited test {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-16T00:18:17,864 INFO  [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/test.com,8080,1, archiveDir=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/oldWALs, maxLogs=32
2024-12-16T00:18:17,864 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1734308297864
2024-12-16T00:18:17,869 INFO  [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/test.com,8080,1/test.com%2C8080%2C1.1734308297864
2024-12-16T00:18:17,869 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33271:33271),(127.0.0.1/127.0.0.1:46501:46501)]
2024-12-16T00:18:17,869 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1734308297869
2024-12-16T00:18:17,877 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/test.com,8080,1/test.com%2C8080%2C1.1734308297864 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/test.com,8080,1/test.com%2C8080%2C1.1734308297869
2024-12-16T00:18:17,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46501:46501),(127.0.0.1/127.0.0.1:33271:33271)]
2024-12-16T00:18:17,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/test.com,8080,1/test.com%2C8080%2C1.1734308297864 is not closed yet, will try archiving it next time
2024-12-16T00:18:17,880 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/test.com,8080,1
2024-12-16T00:18:17,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741837_1013 (size=93)
2024-12-16T00:18:17,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741837_1013 (size=93)
2024-12-16T00:18:17,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741838_1014 (size=93)
2024-12-16T00:18:17,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741838_1014 (size=93)
2024-12-16T00:18:17,882 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/test.com,8080,1/test.com%2C8080%2C1.1734308297864 to hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/oldWALs/test.com%2C8080%2C1.1734308297864
2024-12-16T00:18:17,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/oldWALs
2024-12-16T00:18:17,884 INFO  [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1734308297869)
2024-12-16T00:18:17,884 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-16T00:18:17,884 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5e7cfd0e to 127.0.0.1:58426
2024-12-16T00:18:17,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:18:17,884 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-16T00:18:17,884 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=573227646, stopped=false
2024-12-16T00:18:17,884 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=b878af803003,42441,1734308296072
2024-12-16T00:18:17,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:18:17,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-16T00:18:17,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:17,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:17,898 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-16T00:18:17,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:18:17,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:18:17,898 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,35041,1734308296215' *****
2024-12-16T00:18:17,899 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-16T00:18:17,899 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-16T00:18:17,899 INFO  [RS:0;b878af803003:35041 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-16T00:18:17,899 INFO  [RS:0;b878af803003:35041 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-16T00:18:17,899 INFO  [RS:0;b878af803003:35041 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-16T00:18:17,899 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-16T00:18:17,899 INFO  [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(3579): Received CLOSE for ca171cc0fba2d9cba9a8626247a8c647
2024-12-16T00:18:17,899 INFO  [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1224): stopping server b878af803003,35041,1734308296215
2024-12-16T00:18:17,899 DEBUG [RS:0;b878af803003:35041 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:18:17,899 INFO  [RS:0;b878af803003:35041 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-16T00:18:17,899 INFO  [RS:0;b878af803003:35041 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-16T00:18:17,899 INFO  [RS:0;b878af803003:35041 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-16T00:18:17,899 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ca171cc0fba2d9cba9a8626247a8c647, disabling compactions & flushes
2024-12-16T00:18:17,899 INFO  [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-16T00:18:17,899 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,899 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,899 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647. after waiting 0 ms
2024-12-16T00:18:17,899 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,899 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ca171cc0fba2d9cba9a8626247a8c647 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-16T00:18:17,899 INFO  [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close
2024-12-16T00:18:17,899 DEBUG [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, ca171cc0fba2d9cba9a8626247a8c647=hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.}
2024-12-16T00:18:17,900 DEBUG [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, ca171cc0fba2d9cba9a8626247a8c647
2024-12-16T00:18:17,900 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-16T00:18:17,900 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-16T00:18:17,900 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-16T00:18:17,900 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-16T00:18:17,900 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-16T00:18:17,900 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB
2024-12-16T00:18:17,914 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/namespace/ca171cc0fba2d9cba9a8626247a8c647/.tmp/info/3cd7dea4b5a34ed09ac071151bfe982f is 45, key is default/info:d/1734308297655/Put/seqid=0
2024-12-16T00:18:17,918 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741839_1015 (size=5037)
2024-12-16T00:18:17,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741839_1015 (size=5037)
2024-12-16T00:18:17,919 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/namespace/ca171cc0fba2d9cba9a8626247a8c647/.tmp/info/3cd7dea4b5a34ed09ac071151bfe982f
2024-12-16T00:18:17,920 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/.tmp/info/09f3cd250b1d41a6918cd95ef5e3d280 is 143, key is hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647./info:regioninfo/1734308297595/Put/seqid=0
2024-12-16T00:18:17,923 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741840_1016 (size=6595)
2024-12-16T00:18:17,923 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741840_1016 (size=6595)
2024-12-16T00:18:17,924 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/namespace/ca171cc0fba2d9cba9a8626247a8c647/.tmp/info/3cd7dea4b5a34ed09ac071151bfe982f as hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/namespace/ca171cc0fba2d9cba9a8626247a8c647/info/3cd7dea4b5a34ed09ac071151bfe982f
2024-12-16T00:18:17,924 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/.tmp/info/09f3cd250b1d41a6918cd95ef5e3d280
2024-12-16T00:18:17,928 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/namespace/ca171cc0fba2d9cba9a8626247a8c647/info/3cd7dea4b5a34ed09ac071151bfe982f, entries=2, sequenceid=6, filesize=4.9 K
2024-12-16T00:18:17,929 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for ca171cc0fba2d9cba9a8626247a8c647 in 30ms, sequenceid=6, compaction requested=false
2024-12-16T00:18:17,929 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-16T00:18:17,933 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/namespace/ca171cc0fba2d9cba9a8626247a8c647/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-16T00:18:17,933 INFO  [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,933 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ca171cc0fba2d9cba9a8626247a8c647:

2024-12-16T00:18:17,934 DEBUG [RS_CLOSE_REGION-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1734308297224.ca171cc0fba2d9cba9a8626247a8c647.
2024-12-16T00:18:17,941 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/.tmp/table/4db7b76fd3574eefba93c484b54211d0 is 51, key is hbase:namespace/table:state/1734308297600/Put/seqid=0
2024-12-16T00:18:17,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741841_1017 (size=5242)
2024-12-16T00:18:17,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741841_1017 (size=5242)
2024-12-16T00:18:17,945 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/.tmp/table/4db7b76fd3574eefba93c484b54211d0
2024-12-16T00:18:17,949 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/.tmp/info/09f3cd250b1d41a6918cd95ef5e3d280 as hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/info/09f3cd250b1d41a6918cd95ef5e3d280
2024-12-16T00:18:17,954 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/info/09f3cd250b1d41a6918cd95ef5e3d280, entries=10, sequenceid=9, filesize=6.4 K
2024-12-16T00:18:17,955 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/.tmp/table/4db7b76fd3574eefba93c484b54211d0 as hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/table/4db7b76fd3574eefba93c484b54211d0
2024-12-16T00:18:17,958 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/table/4db7b76fd3574eefba93c484b54211d0, entries=2, sequenceid=9, filesize=5.1 K
2024-12-16T00:18:17,959 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 59ms, sequenceid=9, compaction requested=false
2024-12-16T00:18:17,959 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-16T00:18:17,964 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1
2024-12-16T00:18:17,964 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-16T00:18:17,964 INFO  [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-16T00:18:17,964 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-16T00:18:17,964 DEBUG [RS_CLOSE_META-regionserver/b878af803003:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-16T00:18:18,100 INFO  [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1250): stopping server b878af803003,35041,1734308296215; all regions closed.
2024-12-16T00:18:18,101 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/b878af803003,35041,1734308296215
2024-12-16T00:18:18,104 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741834_1010 (size=2484)
2024-12-16T00:18:18,104 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741834_1010 (size=2484)
2024-12-16T00:18:18,108 DEBUG [RS:0;b878af803003:35041 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/oldWALs
2024-12-16T00:18:18,108 INFO  [RS:0;b878af803003:35041 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog b878af803003%2C35041%2C1734308296215.meta:.meta(num 1734308297103)
2024-12-16T00:18:18,108 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/WALs/b878af803003,35041,1734308296215
2024-12-16T00:18:18,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741833_1009 (size=1414)
2024-12-16T00:18:18,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741833_1009 (size=1414)
2024-12-16T00:18:18,113 DEBUG [RS:0;b878af803003:35041 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/oldWALs
2024-12-16T00:18:18,113 INFO  [RS:0;b878af803003:35041 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog b878af803003%2C35041%2C1734308296215:(num 1734308296721)
2024-12-16T00:18:18,113 DEBUG [RS:0;b878af803003:35041 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:18:18,113 INFO  [RS:0;b878af803003:35041 {}] regionserver.LeaseManager(133): Closed leases
2024-12-16T00:18:18,113 INFO  [RS:0;b878af803003:35041 {}] hbase.ChoreService(370): Chore service for: regionserver/b878af803003:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-16T00:18:18,114 INFO  [regionserver/b878af803003:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:18:18,114 INFO  [RS:0;b878af803003:35041 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:35041
2024-12-16T00:18:18,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b878af803003,35041,1734308296215
2024-12-16T00:18:18,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-16T00:18:18,131 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b878af803003,35041,1734308296215]
2024-12-16T00:18:18,131 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing b878af803003,35041,1734308296215; numProcessing=1
2024-12-16T00:18:18,139 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/b878af803003,35041,1734308296215 already deleted, retry=false
2024-12-16T00:18:18,140 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; b878af803003,35041,1734308296215 expired; onlineServers=0
2024-12-16T00:18:18,140 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'b878af803003,42441,1734308296072' *****
2024-12-16T00:18:18,140 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-16T00:18:18,140 DEBUG [M:0;b878af803003:42441 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56605b4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b878af803003/172.17.0.2:0
2024-12-16T00:18:18,140 INFO  [M:0;b878af803003:42441 {}] regionserver.HRegionServer(1224): stopping server b878af803003,42441,1734308296072
2024-12-16T00:18:18,140 INFO  [M:0;b878af803003:42441 {}] regionserver.HRegionServer(1250): stopping server b878af803003,42441,1734308296072; all regions closed.
2024-12-16T00:18:18,140 DEBUG [M:0;b878af803003:42441 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-16T00:18:18,140 DEBUG [M:0;b878af803003:42441 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-16T00:18:18,140 DEBUG [M:0;b878af803003:42441 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-16T00:18:18,140 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-16T00:18:18,140 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308296497 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.large.0-1734308296497,5,FailOnTimeoutGroup]
2024-12-16T00:18:18,140 DEBUG [master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308296497 {}] cleaner.HFileCleaner(306): Exit Thread[master/b878af803003:0:becomeActiveMaster-HFileCleaner.small.0-1734308296497,5,FailOnTimeoutGroup]
2024-12-16T00:18:18,140 INFO  [M:0;b878af803003:42441 {}] hbase.ChoreService(370): Chore service for: master/b878af803003:0 had [] on shutdown
2024-12-16T00:18:18,140 DEBUG [M:0;b878af803003:42441 {}] master.HMaster(1733): Stopping service threads
2024-12-16T00:18:18,140 INFO  [M:0;b878af803003:42441 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-16T00:18:18,141 INFO  [M:0;b878af803003:42441 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-16T00:18:18,141 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-16T00:18:18,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-16T00:18:18,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-16T00:18:18,148 DEBUG [M:0;b878af803003:42441 {}] zookeeper.ZKUtil(347): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-16T00:18:18,148 WARN  [M:0;b878af803003:42441 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-16T00:18:18,148 INFO  [M:0;b878af803003:42441 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-16T00:18:18,148 INFO  [M:0;b878af803003:42441 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-16T00:18:18,148 DEBUG [M:0;b878af803003:42441 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-16T00:18:18,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-16T00:18:18,148 INFO  [M:0;b878af803003:42441 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:18,148 DEBUG [M:0;b878af803003:42441 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:18,148 DEBUG [M:0;b878af803003:42441 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-16T00:18:18,148 DEBUG [M:0;b878af803003:42441 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:18,149 INFO  [M:0;b878af803003:42441 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=25.32 KB heapSize=32.31 KB
2024-12-16T00:18:18,162 DEBUG [M:0;b878af803003:42441 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e44a121bf345423f97358f95f51647ce is 82, key is hbase:meta,,1/info:regioninfo/1734308297128/Put/seqid=0
2024-12-16T00:18:18,166 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741842_1018 (size=5672)
2024-12-16T00:18:18,166 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741842_1018 (size=5672)
2024-12-16T00:18:18,166 INFO  [M:0;b878af803003:42441 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e44a121bf345423f97358f95f51647ce
2024-12-16T00:18:18,181 DEBUG [M:0;b878af803003:42441 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/af751c443f8344c2ac5c76c413d8250a is 696, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1734308297641/Put/seqid=0
2024-12-16T00:18:18,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741843_1019 (size=6626)
2024-12-16T00:18:18,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741843_1019 (size=6626)
2024-12-16T00:18:18,186 INFO  [M:0;b878af803003:42441 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.72 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/af751c443f8344c2ac5c76c413d8250a
2024-12-16T00:18:18,201 DEBUG [M:0;b878af803003:42441 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7ebbf42d26ca4ae2b6ee787361c65292 is 69, key is b878af803003,35041,1734308296215/rs:state/1734308296570/Put/seqid=0
2024-12-16T00:18:18,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741844_1020 (size=5156)
2024-12-16T00:18:18,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741844_1020 (size=5156)
2024-12-16T00:18:18,206 INFO  [M:0;b878af803003:42441 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7ebbf42d26ca4ae2b6ee787361c65292
2024-12-16T00:18:18,224 DEBUG [M:0;b878af803003:42441 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/21104e7ae5304773b1d7262036ec3af3 is 52, key is load_balancer_on/state:d/1734308297860/Put/seqid=0
2024-12-16T00:18:18,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741845_1021 (size=5056)
2024-12-16T00:18:18,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741845_1021 (size=5056)
2024-12-16T00:18:18,228 INFO  [M:0;b878af803003:42441 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/21104e7ae5304773b1d7262036ec3af3
2024-12-16T00:18:18,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:18:18,232 INFO  [RS:0;b878af803003:35041 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,35041,1734308296215; zookeeper connection closed.
2024-12-16T00:18:18,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35041-0x1002c3391f70001, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:18:18,232 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@50d51cda {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@50d51cda
2024-12-16T00:18:18,232 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-16T00:18:18,233 DEBUG [M:0;b878af803003:42441 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e44a121bf345423f97358f95f51647ce as hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e44a121bf345423f97358f95f51647ce
2024-12-16T00:18:18,236 INFO  [M:0;b878af803003:42441 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e44a121bf345423f97358f95f51647ce, entries=8, sequenceid=70, filesize=5.5 K
2024-12-16T00:18:18,237 DEBUG [M:0;b878af803003:42441 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/af751c443f8344c2ac5c76c413d8250a as hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/af751c443f8344c2ac5c76c413d8250a
2024-12-16T00:18:18,240 INFO  [M:0;b878af803003:42441 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/af751c443f8344c2ac5c76c413d8250a, entries=8, sequenceid=70, filesize=6.5 K
2024-12-16T00:18:18,241 DEBUG [M:0;b878af803003:42441 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7ebbf42d26ca4ae2b6ee787361c65292 as hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7ebbf42d26ca4ae2b6ee787361c65292
2024-12-16T00:18:18,245 INFO  [M:0;b878af803003:42441 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7ebbf42d26ca4ae2b6ee787361c65292, entries=1, sequenceid=70, filesize=5.0 K
2024-12-16T00:18:18,245 DEBUG [M:0;b878af803003:42441 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/21104e7ae5304773b1d7262036ec3af3 as hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/21104e7ae5304773b1d7262036ec3af3
2024-12-16T00:18:18,249 INFO  [M:0;b878af803003:42441 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43035/user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/21104e7ae5304773b1d7262036ec3af3, entries=1, sequenceid=70, filesize=4.9 K
2024-12-16T00:18:18,250 INFO  [M:0;b878af803003:42441 {}] regionserver.HRegion(3040): Finished flush of dataSize ~25.32 KB/25929, heapSize ~32.25 KB/33024, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 102ms, sequenceid=70, compaction requested=false
2024-12-16T00:18:18,251 INFO  [M:0;b878af803003:42441 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-16T00:18:18,251 DEBUG [M:0;b878af803003:42441 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-16T00:18:18,251 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/ca72fa51-0299-6175-34bc-5d57fa45cd2c/MasterData/WALs/b878af803003,42441,1734308296072
2024-12-16T00:18:18,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741830_1006 (size=31030)
2024-12-16T00:18:18,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36727 is added to blk_1073741830_1006 (size=31030)
2024-12-16T00:18:18,253 INFO  [M:0;b878af803003:42441 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-16T00:18:18,253 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-16T00:18:18,253 INFO  [M:0;b878af803003:42441 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:42441
2024-12-16T00:18:18,262 DEBUG [M:0;b878af803003:42441 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/b878af803003,42441,1734308296072 already deleted, retry=false
2024-12-16T00:18:18,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:18:18,373 INFO  [M:0;b878af803003:42441 {}] regionserver.HRegionServer(1307): Exiting; stopping=b878af803003,42441,1734308296072; zookeeper connection closed.
2024-12-16T00:18:18,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42441-0x1002c3391f70000, quorum=127.0.0.1:58426, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-16T00:18:18,379 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6573ca10{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:18:18,379 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@158dc7e4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:18:18,380 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:18:18,380 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@421d8f5d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:18:18,380 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34c1e0e0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/hadoop.log.dir/,STOPPED}
2024-12-16T00:18:18,383 WARN  [BP-1183266520-172.17.0.2-1734308294502 heartbeating to localhost/127.0.0.1:43035 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:18:18,383 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:18:18,383 WARN  [BP-1183266520-172.17.0.2-1734308294502 heartbeating to localhost/127.0.0.1:43035 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1183266520-172.17.0.2-1734308294502 (Datanode Uuid af7fd491-c695-4eb4-8adf-33848a4d4a98) service to localhost/127.0.0.1:43035
2024-12-16T00:18:18,383 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:18:18,384 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb/dfs/data/data3/current/BP-1183266520-172.17.0.2-1734308294502 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:18:18,384 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb/dfs/data/data4/current/BP-1183266520-172.17.0.2-1734308294502 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:18:18,384 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:18:18,390 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5226d17{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-16T00:18:18,390 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@33561086{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:18:18,391 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:18:18,391 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1328adb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:18:18,391 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29d9a359{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/hadoop.log.dir/,STOPPED}
2024-12-16T00:18:18,393 WARN  [BP-1183266520-172.17.0.2-1734308294502 heartbeating to localhost/127.0.0.1:43035 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-16T00:18:18,393 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-16T00:18:18,393 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-16T00:18:18,393 WARN  [BP-1183266520-172.17.0.2-1734308294502 heartbeating to localhost/127.0.0.1:43035 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1183266520-172.17.0.2-1734308294502 (Datanode Uuid 931090f2-334a-49a1-a6b7-c5ab031fb8ce) service to localhost/127.0.0.1:43035
2024-12-16T00:18:18,394 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb/dfs/data/data1/current/BP-1183266520-172.17.0.2-1734308294502 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:18:18,394 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/cluster_2eed3f62-292a-f7bc-efea-6dfb7fdf32bb/dfs/data/data2/current/BP-1183266520-172.17.0.2-1734308294502 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-16T00:18:18,394 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-16T00:18:18,399 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d2c54de{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-16T00:18:18,399 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6ee33572{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-16T00:18:18,399 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-16T00:18:18,400 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10765324{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-16T00:18:18,400 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@166b3543{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/beeb09f8-ef15-0cc7-e013-7392bd5d51f7/hadoop.log.dir/,STOPPED}
2024-12-16T00:18:18,405 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-16T00:18:18,424 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-16T00:18:18,430 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=153 (was 131) - Thread LEAK? -, OpenFileDescriptor=514 (was 490) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=59 (was 47) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12087 (was 12133)